Use redis.

Author: khustup2

cpp/deeplake_pg/pg_deeplake.cpp

@@ -368,6 +368,12 @@ void init_deeplake()
     constexpr int THREAD_POOL_MULTIPLIER = 8;  // Threads per CPU core for async operations
     deeplake_api::initialize(std::make_shared<pg::logger_adapter>(), THREAD_POOL_MULTIPLIER * base::system_report::cpu_cores());
 
+    const std::string redis_url = base::getenv<std::string>("REDIS_URL", "");
+    if (!redis_url.empty()) {
+        deeplake_api::initialize_redis_cache(redis_url, 86400,
+                                             deeplake_api::metadata_catalog_cache_pattern);
+    }
+
     pg::table_storage::instance(); /// initialize table storage
 
     RegisterXactCallback(deeplake_xact_callback, nullptr);

serverless/docker-compose.yml

@@ -21,6 +21,7 @@ services:
       DEEPLAKE_SYNC_INTERVAL_MS: ${DEEPLAKE_SYNC_INTERVAL_MS:-1000}
       PG_DEEPLAKE_MEMORY_LIMIT_MB: ${PG_DEEPLAKE_MEMORY_LIMIT_MB:-0}
       DEEPLAKE_STARTUP_JITTER_MAX_SECONDS: ${DEEPLAKE_STARTUP_JITTER_MAX_SECONDS:-0}
+      REDIS_URL: redis://redis:6379
     entrypoint: ['/usr/local/bin/deeplake-entrypoint.sh']
     command: ['postgres']
     volumes:
@@ -67,6 +68,7 @@ services:
       DEEPLAKE_SYNC_INTERVAL_MS: ${DEEPLAKE_SYNC_INTERVAL_MS:-1000}
       PG_DEEPLAKE_MEMORY_LIMIT_MB: ${PG_DEEPLAKE_MEMORY_LIMIT_MB:-0}
       DEEPLAKE_STARTUP_JITTER_MAX_SECONDS: ${DEEPLAKE_STARTUP_JITTER_MAX_SECONDS:-0}
+      REDIS_URL: redis://redis:6379
     entrypoint: ['/usr/local/bin/deeplake-entrypoint.sh']
     command: ['postgres']
     volumes:

serverless/helm/pg-deeplake/templates/statefulset.yaml

@@ -134,6 +134,10 @@ spec:
               value: {{ .Values.deeplake.startupJitterMaxSeconds | quote }}
             - name: PGDATA
               value: /var/lib/postgresql/data/pgdata
+            {{- if .Values.redis.enabled }}
+            - name: REDIS_URL
+              value: {{ .Values.redis.url | default (printf "redis://%s-redis:%d" (include "pg-deeplake.fullname" .) (int .Values.redis.service.port)) | quote }}
+            {{- end }}
           {{- if .Values.envFromSecret }}
           envFrom:
             - secretRef:

serverless/helm/pg-deeplake/values.yaml

@@ -185,6 +185,7 @@ haproxy:
 
 redis:
   enabled: true
+  url: ""   # Auto-constructed from service name when empty
   image:
     repository: redis
     tag: "7-alpine"

serverless/scripts/manage.py

@@ -273,6 +273,7 @@ def generate_compose(n, profile="haproxy"):
         lines.append("      DEEPLAKE_SYNC_INTERVAL_MS: ${DEEPLAKE_SYNC_INTERVAL_MS:-1000}")
         lines.append("      PG_DEEPLAKE_MEMORY_LIMIT_MB: ${PG_DEEPLAKE_MEMORY_LIMIT_MB:-0}")
         lines.append("      DEEPLAKE_STARTUP_JITTER_MAX_SECONDS: ${DEEPLAKE_STARTUP_JITTER_MAX_SECONDS:-0}")
+        lines.append("      REDIS_URL: redis://redis:6379")
         lines.append("    entrypoint: ['/usr/local/bin/deeplake-entrypoint.sh']")
         lines.append("    command: ['postgres']")
         lines.append("    volumes:")