build psycopg3 test

.builders/images/helpers.ps1

@@ -38,3 +38,12 @@ function Add-ToPath() {
     $target="$oldPath;$Append"
     [Environment]::SetEnvironmentVariable("Path", $target, [System.EnvironmentVariableTarget]::User)
 }
+
+
+function RunOnVSConsole() {
+    param(
+        [Parameter(Mandatory = $true)][string] $Command
+    )
+    Write-Host "Running $Command"
+    Start-Process -Wait -NoNewWindow "cmd.exe" -ArgumentList "/c ""$Env:VCVARSALL_BAT"" $Env:DD_TARGET_ARCH && $Command"
+}

.builders/images/linux-aarch64/Dockerfile

@@ -104,11 +104,11 @@ RUN \
  --without-debugger \
  --disable-static
 
-# libpq and pg_config as needed by psycopg2
-RUN \
+# libpq and pg_config as needed by psycopg
+RUN yum install -y flex && \
  DOWNLOAD_URL="https://ftp.postgresql.org/pub/source/v{{version}}/postgresql-{{version}}.tar.bz2" \
- VERSION="16.0" \
- SHA256="df9e823eb22330444e1d48e52cc65135a652a6fdb3ce325e3f08549339f51b99" \
+ VERSION="16.9" \
+ SHA256="07c00fb824df0a0c295f249f44691b86e3266753b380c96f633c3311e10bd005" \
  RELATIVE_PATH="postgresql-{{version}}" \
  bash install-from-source.sh --without-readline --with-openssl --without-icu
 # Add paths to pg_config and to the library

.builders/images/linux-aarch64/build_script.sh

@@ -29,7 +29,8 @@ if [[ "${DD_BUILD_PYTHON_VERSION}" == "3" ]]; then
     # The version of pyodbc is dynamically linked against a version of the odbc which doesn't come included in the wheel
     # That causes the omnibus' health check to flag it. Forcing the build so that we do include it in the wheel.
     always_build+=("pyodbc")
-
+    # Build psycopg to have control over openssl version
+    always_build+=("psycopg")
     # We need to build cryptography for FIPS support
     always_build+=("cryptography")
 else

.builders/images/linux-x86_64/Dockerfile

@@ -109,10 +109,10 @@ RUN \
  --disable-static
 
 # libpq and pg_config as needed by psycopg2
-RUN \
+RUN yum install -y flex && \
  DOWNLOAD_URL="https://ftp.postgresql.org/pub/source/v{{version}}/postgresql-{{version}}.tar.bz2" \
- VERSION="16.0" \
- SHA256="df9e823eb22330444e1d48e52cc65135a652a6fdb3ce325e3f08549339f51b99" \
+ VERSION="16.9" \
+ SHA256="07c00fb824df0a0c295f249f44691b86e3266753b380c96f633c3311e10bd005" \
  RELATIVE_PATH="postgresql-{{version}}" \
  bash install-from-source.sh --without-readline --with-openssl --without-icu
 # Add paths to pg_config and to the library

.builders/images/linux-x86_64/build_script.sh

@@ -26,7 +26,9 @@ if [[ "${DD_BUILD_PYTHON_VERSION}" == "3" ]]; then
     # The version of pyodbc is dynamically linked against a version of the odbc which doesn't come included in the wheel
     # That causes the omnibus' health check to flag it. Forcing the build so that we do include it in the wheel.
     always_build+=("pyodbc")
-
+    # Build psycopg to have control over openssl version
+    always_build+=("psycopg-c")
+
     # We need to build cryptography for FIPS support
     always_build+=("cryptography")
 fi

.builders/images/macos/builder_setup.sh

@@ -94,6 +94,15 @@ RELATIVE_PATH="curl-{{version}}" \
 # Remove the binary installed so that we consistenly use the same original `curl` binary
 rm "${DD_PREFIX_PATH}/bin/curl"
 
+# libpq and pg_config as needed by psycopg
+DOWNLOAD_URL="https://ftp.postgresql.org/pub/source/v{{version}}/postgresql-{{version}}.tar.bz2" \
+VERSION="16.9" \
+SHA256="07c00fb824df0a0c295f249f44691b86e3266753b380c96f633c3311e10bd005" \
+RELATIVE_PATH="postgresql-{{version}}" \
+  install-from-source --without-readline --with-openssl --without-icu
+# Add paths to pg_config and to the library
+echo PATH="${DD_PREFIX_PATH}/bin:${PATH:-}" >> "$DD_ENV_FILE"
+
 # Dependencies needed to build librdkafka (and thus, confluent-kafka) with kerberos support
 DOWNLOAD_URL="https://github.com/LMDB/lmdb/archive/LMDB_{{version}}.tar.gz" \
 VERSION="0.9.29" \

.builders/images/windows-x86_64/Dockerfile

@@ -27,6 +27,8 @@ RUN curl -SL --output vs_buildtools.exe https://download.visualstudio.microsoft.
      || IF "%ERRORLEVEL%"=="3010" EXIT 0) `
  && del /q vs_buildtools.exe
 
+ ENV VCVARSALL_BAT="C:\Program Files (x86)\Microsoft Visual Studio\2022\BuildTools\VC\Auxiliary\Build\vcvarsall.bat"
+
 # Upgrade PowerShell
 ENV POWERSHELL_VERSION="7.4.0"
 RUN curl -SL --output PowerShell-%POWERSHELL_VERSION%-win-x64.msi https://github.com/PowerShell/PowerShell/releases/download/v%POWERSHELL_VERSION%/PowerShell-%POWERSHELL_VERSION%-win-x64.msi `
@@ -115,7 +117,42 @@ RUN Get-RemoteFile `
     Add-ToPath -Append "C:\perl\perl\bin" && `
     Remove-Item "strawberry-perl-$Env:PERL_VERSION-64bit.zip"
 
+# Nasm
+ENV NASM_VERSION="2.16.03"
+RUN Get-RemoteFile `
+      -Uri https://www.nasm.us/pub/nasm/releasebuilds/$Env:NASM_VERSION/win64/nasm-$Env:NASM_VERSION-win64.zip `
+      -Path "nasm-$Env:NASM_VERSION-win64.zip" `
+    -Hash '3ee4782247bcb874378d02f7eab4e294a84d3d15f3f6ee2de2f47a46aa7226e6' && `
+    7z x "nasm-$Env:NASM_VERSION-win64.zip" -o"C:\nasm" && `
+    Add-ToPath -Append "C:\nasm\nasm-$Env:NASM_VERSION" && `
+    Remove-Item "nasm-$Env:NASM_VERSION-win64.zip"
+
 ENV OPENSSL_VERSION="3.5.2"
+RUN Get-RemoteFile `
+      -Uri https://www.openssl.org/source/openssl-$Env:OPENSSL_VERSION.tar.gz `
+      -Path openssl-$Env:OPENSSL_VERSION.tar.gz `
+      -Hash 'c53a47e5e441c930c3928cf7bf6fb00e5d129b630e0aa873b08258656e7345ec'; `
+    7z x openssl-$Env:OPENSSL_VERSION.tar.gz -r -y && `
+    7z x openssl-$Env:OPENSSL_VERSION.tar -oC:\openssl_3 && `
+    cd C:\openssl_3\openssl-$Env:OPENSSL_VERSION && `
+    RunOnVSConsole -Command `
+    'C:\perl\perl\bin\perl.exe Configure && `
+    nmake && `
+    nmake install_sw'
+# libpq and pg_config as needed by psycopg
+ENV PG_VERSION="16.9"
+COPY pg_config.pl C:\pg_config.pl
+RUN Get-RemoteFile `
+      -Uri https://ftp.postgresql.org/pub/source/v$Env:PG_VERSION/postgresql-$Env:PG_VERSION.tar.bz2 `
+      -Path postgresql-$Env:PG_VERSION.tar.bz2 `
+      -Hash '07c00fb824df0a0c295f249f44691b86e3266753b380c96f633c3311e10bd005'; `
+    7z x postgresql-$Env:PG_VERSION.tar.bz2 -r -y && `
+    7z x postgresql-$Env:PG_VERSION.tar -oC:\postgresql_src && `
+    cd C:\postgresql_src\postgresql-$Env:PG_VERSION\src\tools\msvc && `
+    Copy-Item C:\pg_config.pl -Destination .\config.pl && `
+    RunOnVSConsole -Command 'C:\perl\perl\bin\perl.exe build.pl' && `
+    RunOnVSConsole -Command 'C:\perl\perl\bin\perl.exe install.pl C:\postgresql' && `
+    Add-ToPath -Append "C:\postgresql\bin"
 
 ENV CURL_VERSION="8.15.0"
 

.builders/images/windows-x86_64/pg_config.pl

@@ -0,0 +1,4 @@
+# Overrides over default postgres config
+# https://github.com/postgres/postgres/blob/REL_16_0/src/tools/msvc/config_default.pl
+
+$config->{openssl} = 'c:\Program Files\OpenSSL';

.builders/scripts/repair_wheels.py

@@ -17,7 +17,7 @@
 from utils import extract_metadata, normalize_project_name
 
 # Packages for which we're skipping the openssl-3 build check
-OPENSSL_PACKAGE_BYPASS = ["psycopg"]
+OPENSSL_PACKAGE_BYPASS = []
 
 
 @cache

.ddev/config.toml

@@ -158,7 +158,6 @@ exclude = [
     'aerospike',  # v8+ breaks agent build.
     # https://github.com/DataDog/integrations-core/pull/16080
     'lxml',
-    'psycopg', # Pinning psycopg binary to 3.2.7 to align OpenSSL version with the agent until we build wheels ourselves.
     'psutil',
     'pyvmomi', # 9+ has breaking changes
     'pymongo', # Upgrade from 4.8.0 to 4.10.1 causes "AttributeError: module 'pymongo' has no attribute 'mongo_client'"

.github/workflows/resolve-build-deps.yaml

@@ -246,7 +246,7 @@ jobs:
 
   publish:
     name: Publish artifacts and update lockfiles via PR
-    if: github.event_name == 'push' || (github.event_name == 'workflow_dispatch' && (github.ref_name == github.event.repository.default_branch || startsWith(github.ref_name, '7.')))
+    # if: github.event_name == 'push' || (github.event_name == 'workflow_dispatch' && (github.ref_name == github.event.repository.default_branch || startsWith(github.ref_name, '7.')))
     needs:
     - build
     - build-macos

agent_requirements.in

@@ -33,7 +33,7 @@ ply==3.11
 prometheus-client==0.22.1
 protobuf==6.32.0
 psutil==6.0.0
-psycopg[binary,pool]==3.2.7
+psycopg[c,pool]==3.2.10
 pyasn1==0.4.8
 pycryptodomex==3.23.0
 pydantic==2.11.7

pgbouncer/pyproject.toml

@@ -37,7 +37,7 @@ license = "BSD-3-Clause"
 
 [project.optional-dependencies]
 deps = [
-    "psycopg[binary,pool]==3.2.7",
+    "psycopg[c,pool]==3.2.10",
 ]
 
 [project.urls]

pgbouncer/tests/conftest.py

@@ -15,6 +15,13 @@
 
 HERE = os.path.dirname(os.path.abspath(__file__))
 
+E2E_METADATA = {
+    'start_commands': [
+        'apt update',
+        'apt install -y --no-install-recommends build-essential python3-dev libpq-dev',
+    ],
+}
+
 
 def container_up(service_name, port):
     """
@@ -53,7 +60,7 @@ def dd_environment():
             WaitFor(container_up, args=("PgBouncer", common.PORT)),
         ],
     ):
-        yield common.DEFAULT_INSTANCE
+        yield common.DEFAULT_INSTANCE, E2E_METADATA
 
 
 @pytest.fixture

postgres/pyproject.toml

@@ -40,7 +40,7 @@ deps = [
     "azure-identity==1.24.0",
     "boto3==1.40.21",
     "cachetools==6.2.0",
-    "psycopg[binary,pool]==3.2.7",
+    "psycopg[c,pool]==3.2.10",
     "semver==3.0.4",
 ]
 

silverstripe_cms/pyproject.toml

@@ -37,7 +37,7 @@ dynamic = [
 
 [project.optional-dependencies]
 deps = [
-    "psycopg[binary,pool]==3.2.7",
+    "psycopg[c,pool]==3.2.10",
     "pymysql==1.1.2",
 ]
 

silverstripe_cms/tests/conftest.py

@@ -10,6 +10,13 @@
 
 from .common import COMPOSE, INSTANCE
 
+E2E_METADATA = {
+    'start_commands': [
+        'apt update',
+        'apt install -y --no-install-recommends build-essential python3-dev libpq-dev',
+    ],
+}
+
 
 @pytest.fixture(scope='session')
 def dd_environment():
@@ -23,7 +30,7 @@ def dd_environment():
         sleep=30,
     ):
         instance = INSTANCE.copy()
-        yield instance
+        yield instance, E2E_METADATA
 
 
 @pytest.fixture