Skip to content

Commit bc78d98

Browse files
ArthurKamalovArthurKamalov
authored
[DTIN-2240] Remove dependency packages (#1089)
* Remove Python and Agent libs packages as separate package dependencies and put everything in a sigle final package. * test * simplify OpenSSL selection logic * remove deletion of the pyc files in the package's preuninstall.sh script. * Fix typos
1 parent 28d855e commit bc78d98

34 files changed

+810
-620
lines changed

‎.github/workflows/agent-build-new.yml‎

Lines changed: 48 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2,21 +2,35 @@ name: Agent Build Refactored
22

33
on:
44
push:
5+
branches:
6+
- master
57
pull_request:
6-
# Trigger only for opened pull request. Others have to be handled by the 'push' event
7-
types: [ opened ]
88
branches:
9-
- 'master'
9+
- master
1010
workflow_dispatch:
1111

12-
1312
schedule:
1413
- cron: '0 4 * * *'
1514

1615
permissions:
1716
contents: read
1817

1918
jobs:
19+
pre_job:
20+
name: Skip Duplicate Jobs Pre Job
21+
runs-on: ubuntu-latest
22+
permissions:
23+
actions: write # Needed for skip-duplicate-jobs job
24+
contents: read
25+
outputs:
26+
should_skip: ${{ steps.skip_check.outputs.should_skip }}
27+
steps:
28+
- id: skip_check
29+
uses: fkirc/skip-duplicate-actions@12aca0a884f6137d619d6a8a09fcc3406ced5281 # v4.0.0
30+
with:
31+
cancel_others: 'true'
32+
github_token: ${{ github.token }}
33+
2034
build-linux-packages:
2135
name: "Build Linux packages"
2236
uses: ./.github/workflows/reusable-agent-build-linux-packages-new.yml
@@ -62,3 +76,33 @@ jobs:
6276
status: ${{ job.status }}
6377
steps: ${{ toJson(steps) }}
6478
channel: '#eng-dataset-cloud-tech'
79+
80+
81+
# Aggregate all agent build artifacts that we build by this workflow and put in a single artifact,
82+
# so it's just need to download only one artifact, for example during a release process.
83+
prepare-artifacts:
84+
runs-on: ubuntu-20.04
85+
needs: build-linux-packages
86+
87+
steps:
88+
- name: Checkout repository
89+
uses: actions/checkout@v3
90+
91+
- name: Download artifacts
92+
uses: actions/download-artifact@v3
93+
with:
94+
path: /tmp/all_artifacts
95+
96+
97+
- name: Prepare artifacts
98+
run: |
99+
mkdir -p /tmp/result_artifacts
100+
cp -a /tmp/all_artifacts/linux-packages-*/. /tmp/result_artifacts
101+
cp -a /tmp/all_artifacts/tarball-noarch/. /tmp/result_artifacts
102+
103+
- name: Save result artifact
104+
uses: actions/upload-artifact@v3
105+
with:
106+
name: result-artifacts
107+
path: /tmp/result_artifacts
108+
retention-days: 1

‎.github/workflows/agent-build.yml‎

Lines changed: 22 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,13 @@
11
# The main "setup" workflow, that calls other workflows.
22
name: Docker Images Build
33

4-
on: [push]
4+
on:
5+
push:
6+
branches:
7+
- master
8+
pull_request:
9+
branches:
10+
- master
511

612
permissions:
713
contents: read
@@ -21,6 +27,21 @@ permissions:
2127
#
2228
# For example: https://github.com/scalyr/scalyr-agent-2/pull/804/commits/0eccf278623552b51d9289d75a47794e88f02862
2329
jobs:
30+
pre_job:
31+
name: Skip Duplicate Jobs Pre Job
32+
runs-on: ubuntu-latest
33+
permissions:
34+
actions: write # Needed for skip-duplicate-jobs job
35+
contents: read
36+
outputs:
37+
should_skip: ${{ steps.skip_check.outputs.should_skip }}
38+
steps:
39+
- id: skip_check
40+
uses: fkirc/skip-duplicate-actions@12aca0a884f6137d619d6a8a09fcc3406ced5281 # v4.0.0
41+
with:
42+
cancel_others: 'true'
43+
github_token: ${{ github.token }}
44+
2445
test-images:
2546
runs-on: ubuntu-latest
2647

‎.github/workflows/reusable-agent-build-linux-packages-new.yml‎

Lines changed: 20 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -86,25 +86,25 @@ jobs:
8686
[
8787
{ "name": "deb-x86_64", "master_run_only": false },
8888
{ "name": "rpm-x86_64", "master_run_only": false },
89-
{ "name": "deb-arm64", "master_run_only": true },
90-
{ "name": "rpm-arm64", "master_run_only": true }
89+
{ "name": "deb-arm64", "master_run_only": false },
90+
{ "name": "rpm-arm64", "master_run_only": false }
9191
]
9292
9393
managed-packages-to-test: |
9494
[
95-
{ "name": "deb-x86_64", "distro-name": "ubuntu2204", "remote-machine-type": "ec2", "master_run_only": false },
96-
{ "name": "deb-x86_64", "distro-name": "ubuntu2004", "remote-machine-type": "ec2", "master_run_only": false },
97-
{ "name": "deb-x86_64", "distro-name": "ubuntu1804", "remote-machine-type": "ec2", "master_run_only": false },
98-
{ "name": "deb-x86_64", "distro-name": "ubuntu1604", "remote-machine-type": "ec2", "master_run_only": false },
99-
{ "name": "deb-x86_64", "distro-name": "ubuntu1404", "remote-machine-type": "docker", "master_run_only": false },
100-
{ "name": "deb-x86_64", "distro-name": "debian10", "remote-machine-type": "ec2", "master_run_only": false },
101-
{ "name": "deb-x86_64", "distro-name": "debian11", "remote-machine-type": "docker", "master_run_only": false },
102-
{ "name": "rpm-x86_64", "distro-name": "centos8", "remote-machine-type": "docker", "master_run_only": false },
103-
{ "name": "rpm-x86_64", "distro-name": "centos7", "remote-machine-type": "ec2", "master_run_only": false },
104-
{ "name": "rpm-x86_64", "distro-name": "centos6", "remote-machine-type": "docker", "master_run_only": false },
105-
{ "name": "rpm-x86_64", "distro-name": "amazonlinux2", "remote-machine-type": "ec2", "master_run_only": false },
106-
{ "name": "deb-arm64", "distro-name": "ubuntu1404", "remote-machine-type": "docker", "master_run_only": true },
107-
{ "name": "rpm-arm64", "distro-name": "centos7", "remote-machine-type": "docker", "master_run_only": true }
95+
{ "name": "deb-x86_64", "arch": "x86_64", "distro-name": "ubuntu2204", "remote-machine-type": "ec2", "master_run_only": false },
96+
{ "name": "deb-x86_64", "arch": "x86_64", "distro-name": "ubuntu2004", "remote-machine-type": "ec2", "master_run_only": false },
97+
{ "name": "deb-x86_64", "arch": "x86_64", "distro-name": "ubuntu1804", "remote-machine-type": "ec2", "master_run_only": false },
98+
{ "name": "deb-x86_64", "arch": "x86_64", "distro-name": "ubuntu1604", "remote-machine-type": "ec2", "master_run_only": false },
99+
{ "name": "deb-x86_64", "arch": "x86_64", "distro-name": "ubuntu1404", "remote-machine-type": "docker", "master_run_only": false },
100+
{ "name": "deb-x86_64", "arch": "x86_64", "distro-name": "debian10", "remote-machine-type": "ec2", "master_run_only": false },
101+
{ "name": "deb-x86_64", "arch": "x86_64", "distro-name": "debian11", "remote-machine-type": "docker", "master_run_only": false },
102+
{ "name": "rpm-x86_64", "arch": "x86_64", "distro-name": "centos8", "remote-machine-type": "docker", "master_run_only": false },
103+
{ "name": "rpm-x86_64", "arch": "x86_64", "distro-name": "centos7", "remote-machine-type": "ec2", "master_run_only": false },
104+
{ "name": "rpm-x86_64", "arch": "x86_64", "distro-name": "centos6", "remote-machine-type": "docker", "master_run_only": false },
105+
{ "name": "rpm-x86_64", "arch": "x86_64", "distro-name": "amazonlinux2", "remote-machine-type": "ec2", "master_run_only": false },
106+
{ "name": "deb-arm64", "arch": "arm64", "distro-name": "ubuntu1404", "remote-machine-type": "docker", "master_run_only": false },
107+
{ "name": "rpm-arm64", "arch": "arm64", "distro-name": "centos7", "remote-machine-type": "docker", "master_run_only": false }
108108
]
109109
110110
# This job pre-executes and caches Runner steps that has to be executed in a separate job.
@@ -479,6 +479,11 @@ jobs:
479479
with:
480480
runner-fqdn: ${{ matrix.builder-fqdn }}
481481

482+
- name: Prepare cached steps for package '${{ matrix.name }}' tests
483+
uses: ./.github/actions/execute-runner
484+
with:
485+
runner-fqdn: "tests.end_to_end_tests.run_in_remote_machine.portable_pytest_runner.PortablePytestRunnerBuilder_${{ matrix.arch }}"
486+
482487
- name: Test '${{ matrix.name }}' packages
483488
env:
484489
AWS_ACCESS_KEY: ${{ secrets.CT_AWS_DEV_EC2_ACCESS_KEY }}

‎.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ venv/
1010
env/
1111
.env/
1212
bin/
13+
!agent_build_refactored/managed_packages/files/bin/
1314
build/
1415
cover/
1516
develop-eggs/

‎RELEASE_NOTES.md‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -16,8 +16,8 @@ https://github.com/scalyr/scalyr-agent-2/blob/master/CHANGELOG.md.
1616
seamless for the majority of cases, but if you use custom monitors that require additional libraries, you will need to
1717
specify those requirements in new agent's configuration.
1818

19-
After upgrading to a new version, add required libraries to the file `/opt/scalyr-agent-2-dependencies/etc/additional-requirements.txt`
20-
and run command `/opt/scalyr-agent-2-dependencies/bin/agent-libs-config initialize` to re-initialize requirements.
19+
When upgraded to a new version of the package, add required libraries to the file `/opt/scalyr-agent-2/etc/additional-requirements.txt`
20+
and run command `/opt/scalyr-agent-2/bin/agent-libs-config initialize` to re-initialize requirements.
2121

2222
## 2.1.38 "Zaotune" - Dec 1, 2022
2323

‎agent_build_refactored/managed_packages/build_dependencies_versions.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,3 +31,5 @@
3131

3232
# Version of Rust to use in order to build some of agent's requirements, e.g. orjson.
3333
RUST_VERSION = "1.63.0"
34+
35+
EMBEDDED_PYTHON_PIP_VERSION = "23.0"

‎agent_build_refactored/managed_packages/scalyr_agent_libs/additional-requirements.txt‎ renamed to ‎agent_build_refactored/managed_packages/files/additional-requirements.txt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,2 @@
11
# Here additional requirements for the agent can be specified.
2-
# To install newly added requirements run command '/opt/scalyr-agent-2-dependencies/bin/agent-libs-config initialize'
2+
# To install newly added requirements run command '/opt/scalyr-agent-2/bin/agent-libs-config initialize'

‎agent_build_refactored/managed_packages/scalyr_agent_libs/agent-libs-config‎ renamed to ‎agent_build_refactored/managed_packages/files/bin/agent-libs-config‎

Lines changed: 8 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
#!/opt/scalyr-agent-2-dependencies/bin/python3
1+
#!/opt/scalyr-agent-2/python3/bin/python3
22
# Copyright 2014-2022 Scalyr Inc.
33
#
44
# Licensed under the Apache License, Version 2.0 (the "License");
@@ -21,27 +21,28 @@ import pathlib as pl
2121
import shutil
2222
import subprocess
2323

24-
DEPENDENCIES_PACKAGE_ROOT = pl.Path("/opt/scalyr-agent-2-dependencies")
24+
PACKAGE_OPT_DIR = pl.Path("/opt/scalyr-agent-2")
2525

2626

2727
def initialize():
2828
"""
2929
Initialize agent requirement libraries by creating virtual environment with them.
3030
:return:
3131
"""
32-
core_requirements_path = DEPENDENCIES_PACKAGE_ROOT / "core-requirements.txt"
32+
core_requirements_path = PACKAGE_OPT_DIR / "core-requirements.txt"
3333

3434
# This is pre-created venv that is shipped with package.
35-
venv_source_dir = DEPENDENCIES_PACKAGE_ROOT / "venv"
35+
venv_source_dir = PACKAGE_OPT_DIR / "venv"
3636

37-
venv_dst_dir = pl.Path("/var/opt/scalyr-agent-2-dependencies/venv")
37+
venv_dst_dir = pl.Path("/var/opt/scalyr-agent-2/venv")
3838
if venv_dst_dir.exists():
3939
shutil.rmtree(venv_dst_dir)
4040

4141
# Copy venv that is shipped with the package.
4242
shutil.copytree(
4343
venv_source_dir,
4444
venv_dst_dir,
45+
symlinks=True
4546
)
4647
# Run pip install for additional agent requirements that may be specified by the user.
4748
subprocess.check_call(
@@ -50,11 +51,12 @@ def initialize():
5051
"-m",
5152
"pip",
5253
"install",
54+
"--quiet",
5355
# Core agent requirements are already installed in this "pre-created" venv, but
5456
# we still add their requirements file to be sure that pip will throw an error if
5557
# additional requirements conflict with core ones.
5658
"-r", str(core_requirements_path),
57-
"-r", str(DEPENDENCIES_PACKAGE_ROOT / "etc/additional-requirements.txt"),
59+
"-r", str(PACKAGE_OPT_DIR / "etc/additional-requirements.txt"),
5860
]
5961
)
6062

‎agent_build_refactored/managed_packages/scalyr_agent_python3/internal/agent-python3-config.py‎ renamed to ‎agent_build_refactored/managed_packages/files/bin/agent-python3-config‎

Lines changed: 16 additions & 61 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
#!/opt/scalyr-agent-2/python3/bin/python3
12
# Copyright 2014-2023 Scalyr Inc.
23
#
34
# Licensed under the Apache License, Version 2.0 (the "License");
@@ -13,34 +14,34 @@
1314
# limitations under the License.
1415

1516
"""
16-
This script provide some configuration options for the Agent's Python3 dependency package.
17+
This script provide some configuration options for the Agent's Python3 interpreter.
1718
"""
1819

1920
import argparse
2021
import pathlib as pl
2122
import subprocess
2223
import sys
23-
import re
2424
from typing import Tuple
2525

2626
OPENSSL_1_1_1_VERSION = "1_1_1"
2727
OPENSSL_3_VERSION = "3"
2828

2929
DEFAULT_FALLBACK_OPENSSL_VERSION = OPENSSL_1_1_1_VERSION
3030

31-
DEPENDENCIES_PACKAGE_ROOT = pl.Path("/opt/scalyr-agent-2-dependencies")
32-
OPENSSL_LIBS_PATH = DEPENDENCIES_PACKAGE_ROOT / "lib/openssl"
31+
PACKAGE_OPT_DIR = pl.Path("/opt/scalyr-agent-2")
32+
OPENSSL_LIBS_PATH = PACKAGE_OPT_DIR / "lib/openssl"
33+
PYTHON_DIR = PACKAGE_OPT_DIR / "python3"
3334

3435

3536
PYTHON_SHORT_VERSION = ".".join([str(n) for n in sys.version_info[:2]])
3637

37-
PYTHON_LIB_DYNLOAD_PATH = DEPENDENCIES_PACKAGE_ROOT / f"lib/python{PYTHON_SHORT_VERSION}/lib-dynload"
38+
PYTHON_LIB_DYNLOAD_PATH = PYTHON_DIR / f"lib/python{PYTHON_SHORT_VERSION}/lib-dynload"
3839

3940

40-
PREFERRED_OPENSSL_FILE_PATH = DEPENDENCIES_PACKAGE_ROOT / "etc/preferred_openssl"
41+
PREFERRED_OPENSSL_FILE_PATH = PACKAGE_OPT_DIR / "etc/preferred_openssl"
4142

4243
# Bin directory of the package.
43-
BIN_DIR = DEPENDENCIES_PACKAGE_ROOT / "bin"
44+
PYTHON_BIN_DIR = PYTHON_DIR / "bin"
4445

4546
# This directory is basically a symlink for to another directory with OpenSSL shared objects.
4647
# This script's work is to decide what OpenSLL shared objects to use and link directory with its shared object
@@ -64,10 +65,9 @@ def initialize():
6465

6566
if preferred_openssl == "auto":
6667
try:
67-
unlink_embedded_openssl()
68-
6968
print("Looking for system OpenSSL >= 3: ", file=sys.stderr, end='')
70-
link_bindings(version_type=OPENSSL_3_VERSION)
69+
CURRENT_OPENSSL_LIBS_PATH.unlink()
70+
CURRENT_OPENSSL_LIBS_PATH.symlink_to(f"./{OPENSSL_3_VERSION}")
7171

7272
is_found, version_or_error = get_current_openssl_version()
7373
if is_found:
@@ -77,7 +77,9 @@ def initialize():
7777
print(version_or_error, file=sys.stderr)
7878

7979
print("Looking for system OpenSSL >= 1.1.1: ", file=sys.stderr, end='')
80-
link_bindings(version_type=OPENSSL_1_1_1_VERSION)
80+
CURRENT_OPENSSL_LIBS_PATH.unlink()
81+
CURRENT_OPENSSL_LIBS_PATH.symlink_to(f"./{OPENSSL_1_1_1_VERSION}")
82+
8183
is_found, version_or_error = get_current_openssl_version()
8284
if is_found:
8385
print(f"found {version_or_error}", file=sys.stderr)
@@ -91,8 +93,8 @@ def initialize():
9193
)
9294

9395
print("Using embedded OpenSSL == ", file=sys.stderr, end='')
94-
link_bindings(version_type=OPENSSL_1_1_1_VERSION)
95-
link_embedded_openssl(openssl_variant=OPENSSL_1_1_1_VERSION)
96+
CURRENT_OPENSSL_LIBS_PATH.unlink()
97+
CURRENT_OPENSSL_LIBS_PATH.symlink_to(f"./embedded")
9698
is_found, version_or_error = get_current_openssl_version()
9799
if not is_found:
98100
# Something very wrong happened and this is not expected.
@@ -105,53 +107,6 @@ def initialize():
105107
print(version_or_error, file=sys.stderr)
106108

107109

108-
def unlink_embedded_openssl():
109-
"""
110-
Remove symlink to a directory with embedded OpenSSL shared objects, enabling system's
111-
dynamic linker to look for OpenSSL in other places.
112-
:return:
113-
"""
114-
if CURRENT_OPENSSL_LIBS_PATH.exists():
115-
CURRENT_OPENSSL_LIBS_PATH.unlink()
116-
117-
118-
def link_embedded_openssl(openssl_variant: str):
119-
"""
120-
Create symlink in the path which is included to Python's LD_LIBRARY_PATH variable.
121-
The symlink itself points to the directory with OpenSSL shared objets.
122-
:param openssl_variant:
123-
:return:
124-
"""
125-
unlink_embedded_openssl()
126-
127-
openssl_libs_path = OPENSSL_LIBS_PATH / openssl_variant / "libs"
128-
CURRENT_OPENSSL_LIBS_PATH.symlink_to(openssl_libs_path)
129-
130-
131-
def link_bindings(version_type: str):
132-
"""
133-
Configure Python's 'ssl' and 'hashlib' modules C bindings by linking to their
134-
a particular shared objects.
135-
:param version_type: version of the OpenSSL to use, 1_1_1 or 3
136-
:return:
137-
"""
138-
139-
openssl_version_path = OPENSSL_LIBS_PATH / version_type / "bindings"
140-
141-
for binding_filename_glob in [
142-
"_ssl.*-*-*-*-*.so",
143-
"_hashlib.*-*-*-*-*.so",
144-
]:
145-
binding_path = list(openssl_version_path.glob(binding_filename_glob))[0]
146-
binding_filename = binding_path.name
147-
symlink_path = PYTHON_LIB_DYNLOAD_PATH / binding_filename
148-
149-
if symlink_path.exists():
150-
symlink_path.unlink()
151-
152-
symlink_path.symlink_to(binding_path)
153-
154-
155110
def get_current_openssl_version() -> Tuple[bool, str]:
156111
"""
157112
Determine current system OpenSSL version, if presented.
@@ -163,7 +118,7 @@ def get_current_openssl_version() -> Tuple[bool, str]:
163118

164119
result = subprocess.run(
165120
[
166-
str(BIN_DIR / "python3"),
121+
str(PYTHON_BIN_DIR / "python3"),
167122
"-c",
168123
"import ssl; print(ssl.OPENSSL_VERSION);"
169124
],

0 commit comments

Comments
 (0)