Problem:
DB Migration not completing successfully
Expected outcome:
All containers should start normally
Pulpcore version:
Pulp plugins installed and their versions:
Operating system - distribution and version:
Rocky 8.9 and Docker
Other relevant data:
[root@gsil-docker1 pulp]# cat docker-compose.yml
version: '3'
services:
postgres:
image: "gsil-docker1.idm.gsil.org:5000/postgres:13"
ports:
- "5432:5432"
environment:
POSTGRES_USER: pulp
POSTGRES_PASSWORD: password
POSTGRES_DB: pulp
POSTGRES_INITDB_ARGS: '--auth-host=scram-sha-256'
POSTGRES_HOST_AUTH_METHOD: 'scram-sha-256'
volumes:
- "./pgsql/data:/var/lib/postgresql/data"
- "./assets/postgres/passwd:/etc/passwd:Z"
restart: always
healthcheck:
test: pg_isready -U pulp
interval: 10s
timeout: 5s
retries: 5
redis:
image: "gsil-docker1.idm.gsil.org:5000/redis:latest"
volumes:
- "redis_data:/data"
restart: always
healthcheck:
test: redis-cli ping
interval: 10s
timeout: 5s
retries: 5
migration_service:
image: "gsil-docker1.idm.gsil.org:5000/pulp-minimal:latest"
depends_on:
postgres:
condition: service_healthy
command: pulpcore-manager migrate --noinput
volumes:
- "./assets/settings.py:/etc/pulp/settings.py:z"
- "./assets/certs:/etc/pulp/certs:z"
- "./pulp_storage:/var/lib/pulp"
signing_key_service:
image: "gsil-docker1.idm.gsil.org:5000/pulp-minimal:latest"
command: sh -c "add_signing_service.sh"
depends_on:
postgres:
condition: service_healthy
migration_service:
condition: service_completed_successfully
volumes:
- "./assets/settings.py:/etc/pulp/settings.py:z"
- "./assets/certs:/etc/pulp/certs:z"
- "./pulp_storage:/var/lib/pulp"
set_init_password_service:
image: "gsil-docker1.idm.gsil.org:5000/pulp-minimal:latest"
command: set_init_password.sh
depends_on:
postgres:
condition: service_healthy
environment:
PULP_DEFAULT_ADMIN_PASSWORD: password
volumes:
- "./assets/settings.py:/etc/pulp/settings.py:z"
- "./assets/certs:/etc/pulp/certs:z"
- "./pulp_storage:/var/lib/pulp"
pulp_web:
image: "gsil-docker1.idm.gsil.org:5000/pulp-web:latest"
command: ['/usr/bin/nginx.sh']
depends_on:
pulp_api:
condition: service_healthy
pulp_content:
condition: service_healthy
ports:
- "8081:8080"
hostname: pulp
user: root
volumes:
- "./assets/bin/nginx.sh:/usr/bin/nginx.sh:Z"
- "./assets/nginx/nginx.conf.template:/etc/opt/rh/rh-nginx116/nginx/nginx.conf.template:Z"
restart: always
pulp_api:
image: "gsil-docker1.idm.gsil.org:5000/pulp-minimal:latest"
deploy:
replicas: 2
command: ['pulp-api']
depends_on:
redis:
condition: service_healthy
postgres:
condition: service_healthy
migration_service:
condition: service_completed_successfully
set_init_password_service:
condition: service_completed_successfully
signing_key_service:
condition: service_completed_successfully
hostname: pulp-api
user: pulp
volumes:
- "./assets/settings.py:/etc/pulp/settings.py:z"
- "./assets/certs:/etc/pulp/certs:z"
- "./pulp_storage:/var/lib/pulp"
restart: always
healthcheck:
test: readyz.py /pulp/api/v3/status/
interval: 10s
timeout: 5s
retries: 5
pulp_content:
image: "gsil-docker1.idm.gsil.org:5000/pulp-minimal:latest"
deploy:
replicas: 2
command: ['pulp-content']
depends_on:
redis:
condition: service_healthy
postgres:
condition: service_healthy
migration_service:
condition: service_completed_successfully
hostname: pulp-content
user: pulp
volumes:
- "./assets/settings.py:/etc/pulp/settings.py:z"
- "./assets/certs:/etc/pulp/certs:z"
- "./pulp_storage:/var/lib/pulp"
restart: always
healthcheck:
test: readyz.py /pulp/content/
interval: 10s
timeout: 5s
retries: 5
pulp_worker:
image: "gsil-docker1.idm.gsil.org:5000/pulp-minimal:latest"
deploy:
replicas: 2
command: ['pulp-worker']
depends_on:
redis:
condition: service_healthy
postgres:
condition: service_healthy
migration_service:
condition: service_completed_successfully
user: pulp
volumes:
- "./assets/settings.py:/etc/pulp/settings.py:z"
- "./assets/certs:/etc/pulp/certs:z"
- "./pulp_storage:/var/lib/pulp"
restart: always
volumes:
redis_data:
name: redis_data${DEV_VOLUME_SUFFIX:-dev}
[root@gsil-docker1 pulp]# ls -lah
total 12K
drwx------. 5 root root 79 Jun 11 07:34 .
drwxr-xr-x. 22 root root 4.0K Jun 7 14:15 ..
drwxr-xr-x. 6 localadm localadm 78 Jun 10 08:02 assets
-rw-r--r--. 1 localadm localadm 4.5K Jun 11 07:34 docker-compose.yml
drwxr-xr-x. 3 26 26 18 Jun 11 07:21 pgsql
drwxr-xr-x. 3 700 700 20 Jun 11 07:22 pulp_storage
[root@gsil-docker1 pulp]# docker-compose up -d
[+] Running 68/68
...
[+] Running 13/13
⠿ Network pulp_default Created 0.3s
⠿ Container pulp-postgres-1 Healthy 13.8s
⠿ Container pulp-redis-1 Healthy 13.8s
⠿ Container pulp-migration_service-1 Exited 16.7s
⠿ Container pulp-set_init_password_service-1 Started 12.7s
⠿ Container pulp-pulp_content-2 Created 0.1s
⠿ Container pulp-pulp_worker-2 Created 0.1s
⠿ Container pulp-signing_key_service-1 Created 0.1s
⠿ Container pulp-pulp_content-1 Created 0.1s
⠿ Container pulp-pulp_worker-1 Created 0.1s
⠿ Container pulp-pulp_api-2 Created 0.1s
⠿ Container pulp-pulp_api-1 Created 0.1s
⠿ Container pulp-pulp_web-1 Created 0.0s
service "migration_service" didn't completed successfully: exit 139
pulp-postgres-1 | The files belonging to this database system will be owned by user "postgres".
pulp-postgres-1 | This user must also own the server process.
pulp-postgres-1 |
pulp-postgres-1 | The database cluster will be initialized with locale "en_US.utf8".
pulp-redis-1 | 1:C 11 Jun 2024 12:41:44.727 * oO0OoO0OoO0Oo Redis is starting oO0OoO0OoO0Oo
pulp-redis-1 | 1:C 11 Jun 2024 12:41:44.727 * Redis version=7.2.5, bits=64, commit=00000000, modified=0, pid=1, just started
pulp-redis-1 | 1:C 11 Jun 2024 12:41:44.727 # Warning: no config file specified, using the default config. In order to specify a config file use redis-server /path/to/redis.conf
pulp-redis-1 | 1:M 11 Jun 2024 12:41:44.728 * monotonic clock: POSIX clock_gettime
pulp-redis-1 | 1:M 11 Jun 2024 12:41:44.730 * Running mode=standalone, port=6379.
pulp-postgres-1 | The default database encoding has accordingly been set to "UTF8".
pulp-postgres-1 | The default text search configuration will be set to "english".
pulp-postgres-1 |
pulp-postgres-1 | Data page checksums are disabled.
pulp-postgres-1 |
pulp-postgres-1 | fixing permissions on existing directory /var/lib/postgresql/data ... ok
pulp-postgres-1 | creating subdirectories ... ok
pulp-postgres-1 | selecting dynamic shared memory implementation ... posix
pulp-postgres-1 | selecting default max_connections ... 100
pulp-postgres-1 | selecting default shared_buffers ... 128MB
pulp-postgres-1 | selecting default time zone ... Etc/UTC
pulp-postgres-1 | creating configuration files ... ok
pulp-postgres-1 | running bootstrap script ... ok
pulp-postgres-1 | performing post-bootstrap initialization ... ok
pulp-postgres-1 | initdb: warning: enabling "trust" authentication for local connections
pulp-postgres-1 | You can change this by editing pg_hba.conf or using the option -A, or
pulp-postgres-1 | --auth-local and --auth-host, the next time you run initdb.
pulp-postgres-1 | syncing data to disk ... ok
pulp-postgres-1 |
pulp-postgres-1 |
pulp-postgres-1 | Success. You can now start the database server using:
pulp-postgres-1 |
pulp-postgres-1 | pg_ctl -D /var/lib/postgresql/data -l logfile start
pulp-postgres-1 |
pulp-postgres-1 | waiting for server to start....2024-06-11 12:41:46.191 UTC [48] LOG: starting PostgreSQL 13.15 (Debian 13.15-1.pgdg120+1) on x86_64-pc-linux-gnu, compiled by gcc (Debian 12.2.0-14) 12.2.0, 64-bit
pulp-postgres-1 | 2024-06-11 12:41:46.192 UTC [48] LOG: listening on Unix socket "/var/run/postgresql/.s.PGSQL.5432"
pulp-postgres-1 | 2024-06-11 12:41:46.216 UTC [49] LOG: database system was shut down at 2024-06-11 12:41:46 UTC
pulp-postgres-1 | 2024-06-11 12:41:46.221 UTC [48] LOG: database system is ready to accept connections
pulp-postgres-1 | done
pulp-postgres-1 | server started
pulp-postgres-1 | CREATE DATABASE
pulp-postgres-1 |
pulp-postgres-1 |
pulp-postgres-1 | /usr/local/bin/docker-entrypoint.sh: ignoring /docker-entrypoint-initdb.d/*
pulp-postgres-1 |
pulp-postgres-1 | 2024-06-11 12:41:46.755 UTC [48] LOG: received fast shutdown request
pulp-postgres-1 | waiting for server to shut down....2024-06-11 12:41:46.756 UTC [48] LOG: aborting any active transactions
pulp-postgres-1 | 2024-06-11 12:41:46.758 UTC [48] LOG: background worker "logical replication launcher" (PID 55) exited with exit code 1
pulp-postgres-1 | 2024-06-11 12:41:46.759 UTC [50] LOG: shutting down
pulp-postgres-1 | 2024-06-11 12:41:46.770 UTC [48] LOG: database system is shut down
pulp-postgres-1 | done
pulp-postgres-1 | server stopped
pulp-postgres-1 |
pulp-postgres-1 | PostgreSQL init process complete; ready for start up.
pulp-postgres-1 |
pulp-postgres-1 | 2024-06-11 12:41:46.905 UTC [1] LOG: starting PostgreSQL 13.15 (Debian 13.15-1.pgdg120+1) on x86_64-pc-linux-gnu, compiled by gcc (Debian 12.2.0-14) 12.2.0, 64-bit
pulp-postgres-1 | 2024-06-11 12:41:46.905 UTC [1] LOG: listening on IPv4 address "0.0.0.0", port 5432
pulp-postgres-1 | 2024-06-11 12:41:46.905 UTC [1] LOG: listening on IPv6 address "::", port 5432
pulp-postgres-1 | 2024-06-11 12:41:46.907 UTC [1] LOG: listening on Unix socket "/var/run/postgresql/.s.PGSQL.5432"
pulp-postgres-1 | 2024-06-11 12:41:46.911 UTC [63] LOG: database system was shut down at 2024-06-11 12:41:46 UTC
pulp-postgres-1 | 2024-06-11 12:41:46.916 UTC [1] LOG: database system is ready to accept connections
pulp-redis-1 | 1:M 11 Jun 2024 12:41:44.730 # WARNING: The TCP backlog setting of 511 cannot be enforced because /proc/sys/net/core/somaxconn is set to the lower value of 128.
pulp-redis-1 | 1:M 11 Jun 2024 12:41:44.731 * Server initialized
pulp-redis-1 | 1:M 11 Jun 2024 12:41:44.753 * Loading RDB produced by version 7.2.5
pulp-redis-1 | 1:M 11 Jun 2024 12:41:44.753 * RDB age 214 seconds
pulp-redis-1 | 1:M 11 Jun 2024 12:41:44.753 * RDB memory usage when created 0.85 Mb
pulp-redis-1 | 1:M 11 Jun 2024 12:41:44.753 * Done loading RDB, keys loaded: 0, keys expired: 0.
pulp-redis-1 | 1:M 11 Jun 2024 12:41:44.753 * DB loaded from disk: 0.023 seconds
pulp-redis-1 | 1:M 11 Jun 2024 12:41:44.753 * Ready to accept connections tcp
pulp-postgres-1 | 2024-06-11 12:41:58.226 UTC [78] ERROR: relation "core_artifact" does not exist at character 28
pulp-postgres-1 | 2024-06-11 12:41:58.226 UTC [78] STATEMENT: SELECT count(pulp_id) FROM core_artifact WHERE sha224 IS NULL
pulp-migration_service-1 exited with code 139
pulp-postgres-1 | 2024-06-11 12:41:59.373 UTC [80] ERROR: relation "core_artifact" does not exist at character 28
pulp-postgres-1 | 2024-06-11 12:41:59.373 UTC [80] STATEMENT: SELECT count(pulp_id) FROM core_artifact WHERE sha224 IS NULL
pulp-postgres-1 | 2024-06-11 12:42:00.849 UTC [81] ERROR: relation "core_artifact" does not exist at character 28
pulp-postgres-1 | 2024-06-11 12:42:00.849 UTC [81] STATEMENT: SELECT count(pulp_id) FROM core_artifact WHERE sha224 IS NULL
pulp-set_init_password_service-1 exited with code 134