compose: ingestor + dfe-worker + osrm (grafo baixado 1x para volume)

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Hpjc1TdaRAHoYC6wCFw19E
2026-09-16 19:37:56 -03:00
parent beeea15635
commit e7e43c14f8
3 changed files with 262 additions and 2 deletions
@@ -90,3 +90,7 @@ TERADATA_PASSWORD=
 # ---- Prova do dossiê (backend) ----
 # "1:<openssl rand -base64 32>". PERDER A CHAVE = PERDER AS PROVAS: guarde cópia fora do servidor.
 EVIDENCE_KEYS=
+
+# ---- OSRM (grafo do Brasil, 15 GB) ----
+# Endereço de onde o serviço osrm-data baixa o grafo na 1ª subida (fornecido na hora; temporário).
+OSRM_DATA_URL=
@@ -11,8 +11,10 @@
 #   • Teradata SEFAZ (processor, backend, analytics): nós COP por IP — o DNS da UTIC não
 #     resolve trdtdbcopN. TERADATA_ENABLED/USER/PASSWORD e EVIDENCE_KEYS em Environment
 #     Variables do Coolify. EVIDENCE_KEYS: "1:<openssl rand -base64 32>" (guarde cópia!).
-#   • Certificado A1 (XML sob demanda no backend): DFE_CERT_PFX_BASE64 + DFE_CERT_PASSPHRASE
+#   • Certificado A1 (ingestor, dfe-worker, backend): DFE_CERT_PFX_BASE64 + DFE_CERT_PASSPHRASE
 #     como variáveis secretas no Coolify — sem montar arquivo no servidor.
+#   • Drenagem: ingestor (ONE) + dfe-worker (cadeia) + osrm. O grafo (15 GB) é baixado 1 vez
+#     pelo osrm-data para o volume osrm_data (OSRM_DATA_URL nas Environment Variables).
 # ============================================================================
 x-app-env:
  DATABASE_URL: 'postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@db:5432/${POSTGRES_DB}?schema=public'
@@ -71,6 +73,90 @@ services:
      - gotenberg
      - '--chromium-disable-javascript=false'
      - '--api-timeout=60s'
+  osrm-data:
+    # Baixa UMA VEZ o grafo OSRM do Brasil (15 GB, pré-processado) para o volume osrm_data,
+    # confere o sha256 de cada arquivo e marca .pronto. Próximos deploys: sai na hora.
+    # OSRM_DATA_URL (Environment Variables) = endereço temporário de onde baixar.
+    image: 'gitea.externo.utic.app.br/ericoalmeida/curl:8.15.0'
+    restart: 'no'
+    user: root
+    environment:
+      OSRM_DATA_URL: '${OSRM_DATA_URL}'
+    volumes:
+      - 'osrm_data:/data'
+    entrypoint:
+      - sh
+      - '-c'
+      - |
+        set -e
+        cd /data
+        if [ -f .pronto ]; then echo "grafo OSRM já presente"; exit 0; fi
+        if [ -z "$$OSRM_DATA_URL" ]; then echo "defina OSRM_DATA_URL"; exit 1; fi
+        curl -fsS --retry 10 --retry-all-errors "$$OSRM_DATA_URL/SHA256SUMS" -o SHA256SUMS
+        for f in $$(awk '{print $$2}' SHA256SUMS); do
+          echo "baixando $$f"
+          curl -fS --retry 20 --retry-all-errors --retry-delay 5 -C - "$$OSRM_DATA_URL/$$f" -o "$$f"
+        done
+        sha256sum -c SHA256SUMS
+        touch .pronto
+        echo "grafo OSRM pronto"
+  osrm:
+    # Rotas pelas vias (grafo do Brasil, MLD, v5.26.0). --mmap: lê o grafo do disco sob
+    # demanda (~70-700 MB de RAM em vez de ~15 GB). Leva ~40 s para aceitar rotas.
+    image: 'gitea.externo.utic.app.br/ericoalmeida/osrm-backend:latest'
+    restart: unless-stopped
+    depends_on:
+      osrm-data:
+        condition: service_completed_successfully
+    command:
+      - osrm-routed
+      - '--algorithm'
+      - mld
+      - '--mmap'
+      - '--max-matching-size'
+      - '2000'
+      - /data/brazil.osrm
+    volumes:
+      - 'osrm_data:/data:ro'
+    expose:
+      - '5000'
+  ingestor:
+    # DRENAGEM do ONE (passagens + fotos). Valores de cadência da produção antiga.
+    image: 'gitea.externo.utic.app.br/ericoalmeida/rastro-node:v3.2'
+    restart: unless-stopped
+    depends_on:
+      db:
+        condition: service_healthy
+      rabbitmq:
+        condition: service_healthy
+    environment:
+      DATABASE_URL: 'postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@db:5432/${POSTGRES_DB}?schema=public'
+      RABBITMQ_URL: 'amqp://${RABBITMQ_DEFAULT_USER}:${RABBITMQ_DEFAULT_PASS}@rabbitmq:5672'
+      RASTRO_EXCHANGE: rastro.passagens
+      RASTRO_QUEUE: rastro.passagens.processar
+      NODE_ENV: production
+      DB_CONNECTION_LIMIT: '5'
+      ONE_TP_AMB: '1'
+      ONE_VER_APLIC: rastro-1.0
+      ONE_HOST: one.svrs.rs.gov.br
+      ONE_TIMEOUT_MS: '30000'
+      ONE_TP_DIST: '1,4'
+      ONE_CUF: '41'
+      ONE_CNPJ_OPER: ''
+      ONE_POLL_INTERVAL: '30'
+      ONE_MAX_BATCHES: '40'
+      ONE_BATCH_DELAY_MS: '700'
+      ONE_IND_RESUMO: '0'
+      # Certificado A1 por variável secreta (base64 do .pfx) — sem montar arquivo.
+      DFE_CERT_PFX_BASE64: '${DFE_CERT_PFX_BASE64}'
+      DFE_CERT_PASSPHRASE: '${DFE_CERT_PASSPHRASE}'
+      DFE_TLS_REJECT_UNAUTHORIZED: 'false'
+      MINIO_ENDPOINT: minio
+      MINIO_PORT: '9000'
+      MINIO_ROOT_USER: '${MINIO_ROOT_USER}'
+      MINIO_ROOT_PASSWORD: '${MINIO_ROOT_PASSWORD}'
+      MINIO_BUCKET: rastro-photos
+    command: 'sh -c "wait-for-db schema rabbitmq && pnpm --filter @rastro/ingestor start"'
  migrate:
    image: 'gitea.externo.utic.app.br/ericoalmeida/rastro-node:v3.2'
    restart: 'no'
@@ -112,6 +198,47 @@ services:
      DFE_META_RETENTION_DAYS: '400'
      DFE_LIFECYCLE_ENABLED: 'true'
    command: 'sh -c "wait-for-db schema rabbitmq && pnpm --filter @rastro/processor start"'
+  dfe-worker:
+    # Cadeia MDF-e → CT-e → NF-e (SEFAZ; Teradata para transporte além de D+1) + rota OSRM.
+    image: 'gitea.externo.utic.app.br/ericoalmeida/rastro-node:v3.2'
+    restart: unless-stopped
+    depends_on:
+      db:
+        condition: service_healthy
+    environment:
+      DATABASE_URL: 'postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@db:5432/${POSTGRES_DB}?schema=public'
+      RABBITMQ_URL: 'amqp://${RABBITMQ_DEFAULT_USER}:${RABBITMQ_DEFAULT_PASS}@rabbitmq:5672'
+      RASTRO_EXCHANGE: rastro.passagens
+      RASTRO_QUEUE: rastro.passagens.processar
+      NODE_ENV: production
+      DB_CONNECTION_LIMIT: '12'
+      OSRM_URL: 'http://osrm:5000'
+      DFE_GEN_INTERVAL: '15'
+      DFE_GEN_BATCH: '6'
+      DFE_GEN_CONCURRENCY: '6'
+      DFE_REFRESH_TTL_HOURS: '12'
+      DFE_GEN_ROUTE: 'true'
+      SEFAZ_WS_TP_AMB: '1'
+      SEFAZ_WS_MAX_CTE: '20'
+      SEFAZ_ARQUIVOXML_CUF: '41'
+      SEFAZ_ARQUIVOXML_ORGAO: SEFA-PARANA
+      SEFAZ_ARQUIVOXML_SETOR: AGAA
+      SEFAZ_ARQUIVOXML_VERAPLIC: PR_Rastro_v1
+      SEFAZ_ARQUIVOXML_MATR: '999999'
+      SEFAZ_ARQUIVOXML_MAX_NFE: '40'
+      # Certificado A1 por variável secreta (base64 do .pfx) — sem montar arquivo.
+      DFE_CERT_PFX_BASE64: '${DFE_CERT_PFX_BASE64}'
+      DFE_CERT_PASSPHRASE: '${DFE_CERT_PASSPHRASE}'
+      DFE_TLS_REJECT_UNAUTHORIZED: 'false'
+      TERADATA_ENABLED: '${TERADATA_ENABLED:-false}'
+      TERADATA_HOSTS: '10.14.203.21,10.14.203.22,10.14.203.23'
+      TERADATA_USER: '${TERADATA_USER}'
+      TERADATA_PASSWORD: '${TERADATA_PASSWORD}'
+      TERADATA_DATABASE: P_ACCDB
+      DFE_HOT_DAYS: '7'
+      DFE_OPEN_MAX_DAYS: '60'
+      TD_CHAIN_MIN_AGE_HOURS: '36'
+    command: 'sh -c "wait-for-db schema && pnpm --filter @rastro/processor start:worker"'
  analytics:
    image: 'gitea.externo.utic.app.br/ericoalmeida/rastro-analytics:v4'
    restart: unless-stopped
@@ -205,6 +332,7 @@ services:
    expose:
      - '39020'
 volumes:
+  osrm_data: null
  dbdata2: null
  rabbitmq_data: null
  minio_data: null
@@ -11,8 +11,10 @@
 #   • Teradata SEFAZ (processor, backend, analytics): nós COP por IP — o DNS da UTIC não
 #     resolve trdtdbcopN. TERADATA_ENABLED/USER/PASSWORD e EVIDENCE_KEYS em Environment
 #     Variables do Coolify. EVIDENCE_KEYS: "1:<openssl rand -base64 32>" (guarde cópia!).
-#   • Certificado A1 (XML sob demanda no backend): DFE_CERT_PFX_BASE64 + DFE_CERT_PASSPHRASE
+#   • Certificado A1 (ingestor, dfe-worker, backend): DFE_CERT_PFX_BASE64 + DFE_CERT_PASSPHRASE
 #     como variáveis secretas no Coolify — sem montar arquivo no servidor.
+#   • Drenagem: ingestor (ONE) + dfe-worker (cadeia) + osrm. O grafo (15 GB) é baixado 1 vez
+#     pelo osrm-data para o volume osrm_data (OSRM_DATA_URL nas Environment Variables).
 # ============================================================================
 x-app-env:
  DATABASE_URL: 'postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@db:5432/${POSTGRES_DB}?schema=public'
@@ -71,6 +73,90 @@ services:
      - gotenberg
      - '--chromium-disable-javascript=false'
      - '--api-timeout=60s'
+  osrm-data:
+    # Baixa UMA VEZ o grafo OSRM do Brasil (15 GB, pré-processado) para o volume osrm_data,
+    # confere o sha256 de cada arquivo e marca .pronto. Próximos deploys: sai na hora.
+    # OSRM_DATA_URL (Environment Variables) = endereço temporário de onde baixar.
+    image: 'gitea.externo.utic.app.br/ericoalmeida/curl:8.15.0'
+    restart: 'no'
+    user: root
+    environment:
+      OSRM_DATA_URL: '${OSRM_DATA_URL}'
+    volumes:
+      - 'osrm_data:/data'
+    entrypoint:
+      - sh
+      - '-c'
+      - |
+        set -e
+        cd /data
+        if [ -f .pronto ]; then echo "grafo OSRM já presente"; exit 0; fi
+        if [ -z "$$OSRM_DATA_URL" ]; then echo "defina OSRM_DATA_URL"; exit 1; fi
+        curl -fsS --retry 10 --retry-all-errors "$$OSRM_DATA_URL/SHA256SUMS" -o SHA256SUMS
+        for f in $$(awk '{print $$2}' SHA256SUMS); do
+          echo "baixando $$f"
+          curl -fS --retry 20 --retry-all-errors --retry-delay 5 -C - "$$OSRM_DATA_URL/$$f" -o "$$f"
+        done
+        sha256sum -c SHA256SUMS
+        touch .pronto
+        echo "grafo OSRM pronto"
+  osrm:
+    # Rotas pelas vias (grafo do Brasil, MLD, v5.26.0). --mmap: lê o grafo do disco sob
+    # demanda (~70-700 MB de RAM em vez de ~15 GB). Leva ~40 s para aceitar rotas.
+    image: 'gitea.externo.utic.app.br/ericoalmeida/osrm-backend:latest'
+    restart: unless-stopped
+    depends_on:
+      osrm-data:
+        condition: service_completed_successfully
+    command:
+      - osrm-routed
+      - '--algorithm'
+      - mld
+      - '--mmap'
+      - '--max-matching-size'
+      - '2000'
+      - /data/brazil.osrm
+    volumes:
+      - 'osrm_data:/data:ro'
+    expose:
+      - '5000'
+  ingestor:
+    # DRENAGEM do ONE (passagens + fotos). Valores de cadência da produção antiga.
+    image: 'gitea.externo.utic.app.br/ericoalmeida/rastro-node:v3.2'
+    restart: unless-stopped
+    depends_on:
+      db:
+        condition: service_healthy
+      rabbitmq:
+        condition: service_healthy
+    environment:
+      DATABASE_URL: 'postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@db:5432/${POSTGRES_DB}?schema=public'
+      RABBITMQ_URL: 'amqp://${RABBITMQ_DEFAULT_USER}:${RABBITMQ_DEFAULT_PASS}@rabbitmq:5672'
+      RASTRO_EXCHANGE: rastro.passagens
+      RASTRO_QUEUE: rastro.passagens.processar
+      NODE_ENV: production
+      DB_CONNECTION_LIMIT: '5'
+      ONE_TP_AMB: '1'
+      ONE_VER_APLIC: rastro-1.0
+      ONE_HOST: one.svrs.rs.gov.br
+      ONE_TIMEOUT_MS: '30000'
+      ONE_TP_DIST: '1,4'
+      ONE_CUF: '41'
+      ONE_CNPJ_OPER: ''
+      ONE_POLL_INTERVAL: '30'
+      ONE_MAX_BATCHES: '40'
+      ONE_BATCH_DELAY_MS: '700'
+      ONE_IND_RESUMO: '0'
+      # Certificado A1 por variável secreta (base64 do .pfx) — sem montar arquivo.
+      DFE_CERT_PFX_BASE64: '${DFE_CERT_PFX_BASE64}'
+      DFE_CERT_PASSPHRASE: '${DFE_CERT_PASSPHRASE}'
+      DFE_TLS_REJECT_UNAUTHORIZED: 'false'
+      MINIO_ENDPOINT: minio
+      MINIO_PORT: '9000'
+      MINIO_ROOT_USER: '${MINIO_ROOT_USER}'
+      MINIO_ROOT_PASSWORD: '${MINIO_ROOT_PASSWORD}'
+      MINIO_BUCKET: rastro-photos
+    command: 'sh -c "wait-for-db schema rabbitmq && pnpm --filter @rastro/ingestor start"'
  migrate:
    image: 'gitea.externo.utic.app.br/ericoalmeida/rastro-node:v3.2'
    restart: 'no'
@@ -112,6 +198,47 @@ services:
      DFE_META_RETENTION_DAYS: '400'
      DFE_LIFECYCLE_ENABLED: 'true'
    command: 'sh -c "wait-for-db schema rabbitmq && pnpm --filter @rastro/processor start"'
+  dfe-worker:
+    # Cadeia MDF-e → CT-e → NF-e (SEFAZ; Teradata para transporte além de D+1) + rota OSRM.
+    image: 'gitea.externo.utic.app.br/ericoalmeida/rastro-node:v3.2'
+    restart: unless-stopped
+    depends_on:
+      db:
+        condition: service_healthy
+    environment:
+      DATABASE_URL: 'postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@db:5432/${POSTGRES_DB}?schema=public'
+      RABBITMQ_URL: 'amqp://${RABBITMQ_DEFAULT_USER}:${RABBITMQ_DEFAULT_PASS}@rabbitmq:5672'
+      RASTRO_EXCHANGE: rastro.passagens
+      RASTRO_QUEUE: rastro.passagens.processar
+      NODE_ENV: production
+      DB_CONNECTION_LIMIT: '12'
+      OSRM_URL: 'http://osrm:5000'
+      DFE_GEN_INTERVAL: '15'
+      DFE_GEN_BATCH: '6'
+      DFE_GEN_CONCURRENCY: '6'
+      DFE_REFRESH_TTL_HOURS: '12'
+      DFE_GEN_ROUTE: 'true'
+      SEFAZ_WS_TP_AMB: '1'
+      SEFAZ_WS_MAX_CTE: '20'
+      SEFAZ_ARQUIVOXML_CUF: '41'
+      SEFAZ_ARQUIVOXML_ORGAO: SEFA-PARANA
+      SEFAZ_ARQUIVOXML_SETOR: AGAA
+      SEFAZ_ARQUIVOXML_VERAPLIC: PR_Rastro_v1
+      SEFAZ_ARQUIVOXML_MATR: '999999'
+      SEFAZ_ARQUIVOXML_MAX_NFE: '40'
+      # Certificado A1 por variável secreta (base64 do .pfx) — sem montar arquivo.
+      DFE_CERT_PFX_BASE64: '${DFE_CERT_PFX_BASE64}'
+      DFE_CERT_PASSPHRASE: '${DFE_CERT_PASSPHRASE}'
+      DFE_TLS_REJECT_UNAUTHORIZED: 'false'
+      TERADATA_ENABLED: '${TERADATA_ENABLED:-false}'
+      TERADATA_HOSTS: '10.14.203.21,10.14.203.22,10.14.203.23'
+      TERADATA_USER: '${TERADATA_USER}'
+      TERADATA_PASSWORD: '${TERADATA_PASSWORD}'
+      TERADATA_DATABASE: P_ACCDB
+      DFE_HOT_DAYS: '7'
+      DFE_OPEN_MAX_DAYS: '60'
+      TD_CHAIN_MIN_AGE_HOURS: '36'
+    command: 'sh -c "wait-for-db schema && pnpm --filter @rastro/processor start:worker"'
  analytics:
    image: 'gitea.externo.utic.app.br/ericoalmeida/rastro-analytics:v4'
    restart: unless-stopped
@@ -205,6 +332,7 @@ services:
    expose:
      - '39020'
 volumes:
+  osrm_data: null
  dbdata2: null
  rabbitmq_data: null
  minio_data: null