diff --git a/.env.example b/.env.example index 2c981bb..bd5aac5 100644 --- a/.env.example +++ b/.env.example @@ -90,3 +90,7 @@ TERADATA_PASSWORD= # ---- Prova do dossiê (backend) ---- # "1:". PERDER A CHAVE = PERDER AS PROVAS: guarde cópia fora do servidor. EVIDENCE_KEYS= + +# ---- OSRM (grafo do Brasil, 15 GB) ---- +# Endereço de onde o serviço osrm-data baixa o grafo na 1ª subida (fornecido na hora; temporário). +OSRM_DATA_URL= diff --git a/docker-compose.yaml b/docker-compose.yaml index 66eed4f..af9c468 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -11,8 +11,10 @@ # • Teradata SEFAZ (processor, backend, analytics): nós COP por IP — o DNS da UTIC não # resolve trdtdbcopN. TERADATA_ENABLED/USER/PASSWORD e EVIDENCE_KEYS em Environment # Variables do Coolify. EVIDENCE_KEYS: "1:" (guarde cópia!). -# • Certificado A1 (XML sob demanda no backend): DFE_CERT_PFX_BASE64 + DFE_CERT_PASSPHRASE +# • Certificado A1 (ingestor, dfe-worker, backend): DFE_CERT_PFX_BASE64 + DFE_CERT_PASSPHRASE # como variáveis secretas no Coolify — sem montar arquivo no servidor. +# • Drenagem: ingestor (ONE) + dfe-worker (cadeia) + osrm. O grafo (15 GB) é baixado 1 vez +# pelo osrm-data para o volume osrm_data (OSRM_DATA_URL nas Environment Variables). # ============================================================================ x-app-env: DATABASE_URL: 'postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@db:5432/${POSTGRES_DB}?schema=public' @@ -71,6 +73,90 @@ services: - gotenberg - '--chromium-disable-javascript=false' - '--api-timeout=60s' + osrm-data: + # Baixa UMA VEZ o grafo OSRM do Brasil (15 GB, pré-processado) para o volume osrm_data, + # confere o sha256 de cada arquivo e marca .pronto. Próximos deploys: sai na hora. + # OSRM_DATA_URL (Environment Variables) = endereço temporário de onde baixar. + image: 'gitea.externo.utic.app.br/ericoalmeida/curl:8.15.0' + restart: 'no' + user: root + environment: + OSRM_DATA_URL: '${OSRM_DATA_URL}' + volumes: + - 'osrm_data:/data' + entrypoint: + - sh + - '-c' + - | + set -e + cd /data + if [ -f .pronto ]; then echo "grafo OSRM já presente"; exit 0; fi + if [ -z "$$OSRM_DATA_URL" ]; then echo "defina OSRM_DATA_URL"; exit 1; fi + curl -fsS --retry 10 --retry-all-errors "$$OSRM_DATA_URL/SHA256SUMS" -o SHA256SUMS + for f in $$(awk '{print $$2}' SHA256SUMS); do + echo "baixando $$f" + curl -fS --retry 20 --retry-all-errors --retry-delay 5 -C - "$$OSRM_DATA_URL/$$f" -o "$$f" + done + sha256sum -c SHA256SUMS + touch .pronto + echo "grafo OSRM pronto" + osrm: + # Rotas pelas vias (grafo do Brasil, MLD, v5.26.0). --mmap: lê o grafo do disco sob + # demanda (~70-700 MB de RAM em vez de ~15 GB). Leva ~40 s para aceitar rotas. + image: 'gitea.externo.utic.app.br/ericoalmeida/osrm-backend:latest' + restart: unless-stopped + depends_on: + osrm-data: + condition: service_completed_successfully + command: + - osrm-routed + - '--algorithm' + - mld + - '--mmap' + - '--max-matching-size' + - '2000' + - /data/brazil.osrm + volumes: + - 'osrm_data:/data:ro' + expose: + - '5000' + ingestor: + # DRENAGEM do ONE (passagens + fotos). Valores de cadência da produção antiga. + image: 'gitea.externo.utic.app.br/ericoalmeida/rastro-node:v3.2' + restart: unless-stopped + depends_on: + db: + condition: service_healthy + rabbitmq: + condition: service_healthy + environment: + DATABASE_URL: 'postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@db:5432/${POSTGRES_DB}?schema=public' + RABBITMQ_URL: 'amqp://${RABBITMQ_DEFAULT_USER}:${RABBITMQ_DEFAULT_PASS}@rabbitmq:5672' + RASTRO_EXCHANGE: rastro.passagens + RASTRO_QUEUE: rastro.passagens.processar + NODE_ENV: production + DB_CONNECTION_LIMIT: '5' + ONE_TP_AMB: '1' + ONE_VER_APLIC: rastro-1.0 + ONE_HOST: one.svrs.rs.gov.br + ONE_TIMEOUT_MS: '30000' + ONE_TP_DIST: '1,4' + ONE_CUF: '41' + ONE_CNPJ_OPER: '' + ONE_POLL_INTERVAL: '30' + ONE_MAX_BATCHES: '40' + ONE_BATCH_DELAY_MS: '700' + ONE_IND_RESUMO: '0' + # Certificado A1 por variável secreta (base64 do .pfx) — sem montar arquivo. + DFE_CERT_PFX_BASE64: '${DFE_CERT_PFX_BASE64}' + DFE_CERT_PASSPHRASE: '${DFE_CERT_PASSPHRASE}' + DFE_TLS_REJECT_UNAUTHORIZED: 'false' + MINIO_ENDPOINT: minio + MINIO_PORT: '9000' + MINIO_ROOT_USER: '${MINIO_ROOT_USER}' + MINIO_ROOT_PASSWORD: '${MINIO_ROOT_PASSWORD}' + MINIO_BUCKET: rastro-photos + command: 'sh -c "wait-for-db schema rabbitmq && pnpm --filter @rastro/ingestor start"' migrate: image: 'gitea.externo.utic.app.br/ericoalmeida/rastro-node:v3.2' restart: 'no' @@ -112,6 +198,47 @@ services: DFE_META_RETENTION_DAYS: '400' DFE_LIFECYCLE_ENABLED: 'true' command: 'sh -c "wait-for-db schema rabbitmq && pnpm --filter @rastro/processor start"' + dfe-worker: + # Cadeia MDF-e → CT-e → NF-e (SEFAZ; Teradata para transporte além de D+1) + rota OSRM. + image: 'gitea.externo.utic.app.br/ericoalmeida/rastro-node:v3.2' + restart: unless-stopped + depends_on: + db: + condition: service_healthy + environment: + DATABASE_URL: 'postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@db:5432/${POSTGRES_DB}?schema=public' + RABBITMQ_URL: 'amqp://${RABBITMQ_DEFAULT_USER}:${RABBITMQ_DEFAULT_PASS}@rabbitmq:5672' + RASTRO_EXCHANGE: rastro.passagens + RASTRO_QUEUE: rastro.passagens.processar + NODE_ENV: production + DB_CONNECTION_LIMIT: '12' + OSRM_URL: 'http://osrm:5000' + DFE_GEN_INTERVAL: '15' + DFE_GEN_BATCH: '6' + DFE_GEN_CONCURRENCY: '6' + DFE_REFRESH_TTL_HOURS: '12' + DFE_GEN_ROUTE: 'true' + SEFAZ_WS_TP_AMB: '1' + SEFAZ_WS_MAX_CTE: '20' + SEFAZ_ARQUIVOXML_CUF: '41' + SEFAZ_ARQUIVOXML_ORGAO: SEFA-PARANA + SEFAZ_ARQUIVOXML_SETOR: AGAA + SEFAZ_ARQUIVOXML_VERAPLIC: PR_Rastro_v1 + SEFAZ_ARQUIVOXML_MATR: '999999' + SEFAZ_ARQUIVOXML_MAX_NFE: '40' + # Certificado A1 por variável secreta (base64 do .pfx) — sem montar arquivo. + DFE_CERT_PFX_BASE64: '${DFE_CERT_PFX_BASE64}' + DFE_CERT_PASSPHRASE: '${DFE_CERT_PASSPHRASE}' + DFE_TLS_REJECT_UNAUTHORIZED: 'false' + TERADATA_ENABLED: '${TERADATA_ENABLED:-false}' + TERADATA_HOSTS: '10.14.203.21,10.14.203.22,10.14.203.23' + TERADATA_USER: '${TERADATA_USER}' + TERADATA_PASSWORD: '${TERADATA_PASSWORD}' + TERADATA_DATABASE: P_ACCDB + DFE_HOT_DAYS: '7' + DFE_OPEN_MAX_DAYS: '60' + TD_CHAIN_MIN_AGE_HOURS: '36' + command: 'sh -c "wait-for-db schema && pnpm --filter @rastro/processor start:worker"' analytics: image: 'gitea.externo.utic.app.br/ericoalmeida/rastro-analytics:v4' restart: unless-stopped @@ -205,6 +332,7 @@ services: expose: - '39020' volumes: + osrm_data: null dbdata2: null rabbitmq_data: null minio_data: null diff --git a/docker-compose.yml b/docker-compose.yml index 66eed4f..af9c468 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -11,8 +11,10 @@ # • Teradata SEFAZ (processor, backend, analytics): nós COP por IP — o DNS da UTIC não # resolve trdtdbcopN. TERADATA_ENABLED/USER/PASSWORD e EVIDENCE_KEYS em Environment # Variables do Coolify. EVIDENCE_KEYS: "1:" (guarde cópia!). -# • Certificado A1 (XML sob demanda no backend): DFE_CERT_PFX_BASE64 + DFE_CERT_PASSPHRASE +# • Certificado A1 (ingestor, dfe-worker, backend): DFE_CERT_PFX_BASE64 + DFE_CERT_PASSPHRASE # como variáveis secretas no Coolify — sem montar arquivo no servidor. +# • Drenagem: ingestor (ONE) + dfe-worker (cadeia) + osrm. O grafo (15 GB) é baixado 1 vez +# pelo osrm-data para o volume osrm_data (OSRM_DATA_URL nas Environment Variables). # ============================================================================ x-app-env: DATABASE_URL: 'postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@db:5432/${POSTGRES_DB}?schema=public' @@ -71,6 +73,90 @@ services: - gotenberg - '--chromium-disable-javascript=false' - '--api-timeout=60s' + osrm-data: + # Baixa UMA VEZ o grafo OSRM do Brasil (15 GB, pré-processado) para o volume osrm_data, + # confere o sha256 de cada arquivo e marca .pronto. Próximos deploys: sai na hora. + # OSRM_DATA_URL (Environment Variables) = endereço temporário de onde baixar. + image: 'gitea.externo.utic.app.br/ericoalmeida/curl:8.15.0' + restart: 'no' + user: root + environment: + OSRM_DATA_URL: '${OSRM_DATA_URL}' + volumes: + - 'osrm_data:/data' + entrypoint: + - sh + - '-c' + - | + set -e + cd /data + if [ -f .pronto ]; then echo "grafo OSRM já presente"; exit 0; fi + if [ -z "$$OSRM_DATA_URL" ]; then echo "defina OSRM_DATA_URL"; exit 1; fi + curl -fsS --retry 10 --retry-all-errors "$$OSRM_DATA_URL/SHA256SUMS" -o SHA256SUMS + for f in $$(awk '{print $$2}' SHA256SUMS); do + echo "baixando $$f" + curl -fS --retry 20 --retry-all-errors --retry-delay 5 -C - "$$OSRM_DATA_URL/$$f" -o "$$f" + done + sha256sum -c SHA256SUMS + touch .pronto + echo "grafo OSRM pronto" + osrm: + # Rotas pelas vias (grafo do Brasil, MLD, v5.26.0). --mmap: lê o grafo do disco sob + # demanda (~70-700 MB de RAM em vez de ~15 GB). Leva ~40 s para aceitar rotas. + image: 'gitea.externo.utic.app.br/ericoalmeida/osrm-backend:latest' + restart: unless-stopped + depends_on: + osrm-data: + condition: service_completed_successfully + command: + - osrm-routed + - '--algorithm' + - mld + - '--mmap' + - '--max-matching-size' + - '2000' + - /data/brazil.osrm + volumes: + - 'osrm_data:/data:ro' + expose: + - '5000' + ingestor: + # DRENAGEM do ONE (passagens + fotos). Valores de cadência da produção antiga. + image: 'gitea.externo.utic.app.br/ericoalmeida/rastro-node:v3.2' + restart: unless-stopped + depends_on: + db: + condition: service_healthy + rabbitmq: + condition: service_healthy + environment: + DATABASE_URL: 'postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@db:5432/${POSTGRES_DB}?schema=public' + RABBITMQ_URL: 'amqp://${RABBITMQ_DEFAULT_USER}:${RABBITMQ_DEFAULT_PASS}@rabbitmq:5672' + RASTRO_EXCHANGE: rastro.passagens + RASTRO_QUEUE: rastro.passagens.processar + NODE_ENV: production + DB_CONNECTION_LIMIT: '5' + ONE_TP_AMB: '1' + ONE_VER_APLIC: rastro-1.0 + ONE_HOST: one.svrs.rs.gov.br + ONE_TIMEOUT_MS: '30000' + ONE_TP_DIST: '1,4' + ONE_CUF: '41' + ONE_CNPJ_OPER: '' + ONE_POLL_INTERVAL: '30' + ONE_MAX_BATCHES: '40' + ONE_BATCH_DELAY_MS: '700' + ONE_IND_RESUMO: '0' + # Certificado A1 por variável secreta (base64 do .pfx) — sem montar arquivo. + DFE_CERT_PFX_BASE64: '${DFE_CERT_PFX_BASE64}' + DFE_CERT_PASSPHRASE: '${DFE_CERT_PASSPHRASE}' + DFE_TLS_REJECT_UNAUTHORIZED: 'false' + MINIO_ENDPOINT: minio + MINIO_PORT: '9000' + MINIO_ROOT_USER: '${MINIO_ROOT_USER}' + MINIO_ROOT_PASSWORD: '${MINIO_ROOT_PASSWORD}' + MINIO_BUCKET: rastro-photos + command: 'sh -c "wait-for-db schema rabbitmq && pnpm --filter @rastro/ingestor start"' migrate: image: 'gitea.externo.utic.app.br/ericoalmeida/rastro-node:v3.2' restart: 'no' @@ -112,6 +198,47 @@ services: DFE_META_RETENTION_DAYS: '400' DFE_LIFECYCLE_ENABLED: 'true' command: 'sh -c "wait-for-db schema rabbitmq && pnpm --filter @rastro/processor start"' + dfe-worker: + # Cadeia MDF-e → CT-e → NF-e (SEFAZ; Teradata para transporte além de D+1) + rota OSRM. + image: 'gitea.externo.utic.app.br/ericoalmeida/rastro-node:v3.2' + restart: unless-stopped + depends_on: + db: + condition: service_healthy + environment: + DATABASE_URL: 'postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@db:5432/${POSTGRES_DB}?schema=public' + RABBITMQ_URL: 'amqp://${RABBITMQ_DEFAULT_USER}:${RABBITMQ_DEFAULT_PASS}@rabbitmq:5672' + RASTRO_EXCHANGE: rastro.passagens + RASTRO_QUEUE: rastro.passagens.processar + NODE_ENV: production + DB_CONNECTION_LIMIT: '12' + OSRM_URL: 'http://osrm:5000' + DFE_GEN_INTERVAL: '15' + DFE_GEN_BATCH: '6' + DFE_GEN_CONCURRENCY: '6' + DFE_REFRESH_TTL_HOURS: '12' + DFE_GEN_ROUTE: 'true' + SEFAZ_WS_TP_AMB: '1' + SEFAZ_WS_MAX_CTE: '20' + SEFAZ_ARQUIVOXML_CUF: '41' + SEFAZ_ARQUIVOXML_ORGAO: SEFA-PARANA + SEFAZ_ARQUIVOXML_SETOR: AGAA + SEFAZ_ARQUIVOXML_VERAPLIC: PR_Rastro_v1 + SEFAZ_ARQUIVOXML_MATR: '999999' + SEFAZ_ARQUIVOXML_MAX_NFE: '40' + # Certificado A1 por variável secreta (base64 do .pfx) — sem montar arquivo. + DFE_CERT_PFX_BASE64: '${DFE_CERT_PFX_BASE64}' + DFE_CERT_PASSPHRASE: '${DFE_CERT_PASSPHRASE}' + DFE_TLS_REJECT_UNAUTHORIZED: 'false' + TERADATA_ENABLED: '${TERADATA_ENABLED:-false}' + TERADATA_HOSTS: '10.14.203.21,10.14.203.22,10.14.203.23' + TERADATA_USER: '${TERADATA_USER}' + TERADATA_PASSWORD: '${TERADATA_PASSWORD}' + TERADATA_DATABASE: P_ACCDB + DFE_HOT_DAYS: '7' + DFE_OPEN_MAX_DAYS: '60' + TD_CHAIN_MIN_AGE_HOURS: '36' + command: 'sh -c "wait-for-db schema && pnpm --filter @rastro/processor start:worker"' analytics: image: 'gitea.externo.utic.app.br/ericoalmeida/rastro-analytics:v4' restart: unless-stopped @@ -205,6 +332,7 @@ services: expose: - '39020' volumes: + osrm_data: null dbdata2: null rabbitmq_data: null minio_data: null