Skip to content

armgit / core

Public Imported from GitHub
0

100% open-source IoT Platform - Integrate your devices, add intelligence, build custom apps

Remove manager storage volume from postgres container and related references (#3288)

Eric Bariaux74df6ad58743
diff --git a/.github/workflows/ci_cd.yml b/.github/workflows/ci_cd.yml
index d0fdd113f..91375cafd 100644
--- a/.github/workflows/ci_cd.yml
+++ b/.github/workflows/ci_cd.yml
@@ -724,9 +724,6 @@ jobs:
run: |
composeProfile='profile/dev-testing.yml'
- # Make temp dir and set mask to 777 as docker seems to run as root
- mkdir -pm 777 tmp
-
# Define cleanup command
echo "cleanup=docker compose -f $composeProfile down -v" >> $GITHUB_OUTPUT
@@ -814,9 +811,6 @@ jobs:
run: |
composeProfile='profile/dev-testing.yml'
- # Make temp dir and set mask to 777 as docker seems to run as root
- mkdir -pm 777 tmp
-
# Define cleanup command
echo "cleanup=docker compose -f $composeProfile down -v" >> $GITHUB_OUTPUT
diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml
index e21871834..78b13a567 100644
--- a/.github/workflows/integration-tests.yml
+++ b/.github/workflows/integration-tests.yml
@@ -28,8 +28,6 @@ jobs:
timeout-minutes: 20
run: |
# Run Integration Tests
- # Make temp dir with 777 mask as docker seems to run as root
- mkdir -pm 777 tmp
# Start the stack
docker compose -f profile/dev-testing.yml up -d --no-build
diff --git a/.github/workflows/ui-app-tests.yml b/.github/workflows/ui-app-tests.yml
index 72b18b7a6..79610951a 100644
--- a/.github/workflows/ui-app-tests.yml
+++ b/.github/workflows/ui-app-tests.yml
@@ -46,9 +46,6 @@ jobs:
- name: Run UI App Tests
timeout-minutes: 20
run: |
- # Make temp dir with 777 mask as docker seems to run as root
- mkdir -pm 777 tmp
-
# Start the stack
docker compose -f profile/dev-testing.yml up -d --no-build
diff --git a/AGENTS.md b/AGENTS.md
index 5c6931e56..46545faba 100644
--- a/AGENTS.md
+++ b/AGENTS.md
@@ -13,8 +13,7 @@ In sandboxed environments, add `--offline` (requires dependencies to already be
## Running tests
-When running integration tests, part of the stack (PostgreSQL and Keycloak) must be running. Start it with `mkdir -pm 777 tmp && docker compose -f profile/dev-testing.yml -p openremote up -d --no-build`.
-Running `./gradlew clean` deletes the root `tmp/` directory that is mounted into PostgreSQL (see `profile/dev-testing.yml`), so recreate it and restart the stack before running tests again.
+When running integration tests, part of the stack (PostgreSQL and Keycloak) must be running. Start it with `docker compose -f profile/dev-testing.yml -p openremote up -d --no-build`.
## REST resources
diff --git a/balena/docker-compose.yml b/balena/docker-compose.yml
index ae1377975..c2552c188 100755
--- a/balena/docker-compose.yml
+++ b/balena/docker-compose.yml
@@ -27,7 +27,6 @@ services:
restart: always
volumes:
- postgresql-data:/var/lib/postgresql/data
- - manager-data:/storage
env_file:
- env
diff --git a/docker-compose.yml b/docker-compose.yml
index c6a9503d0..a0d38e938 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -39,7 +39,6 @@ services:
shm_size: 128mb
volumes:
- postgresql-data:/var/lib/postgresql/data
- - manager-data:/storage
keycloak:
restart: always
diff --git a/manager/src/main/java/org/openremote/manager/datapoint/AssetDatapointService.java b/manager/src/main/java/org/openremote/manager/datapoint/AssetDatapointService.java
index 4fb84114e..47182ba41 100644
--- a/manager/src/main/java/org/openremote/manager/datapoint/AssetDatapointService.java
+++ b/manager/src/main/java/org/openremote/manager/datapoint/AssetDatapointService.java
@@ -282,9 +282,8 @@ public class AssetDatapointService extends AbstractDatapointService<AssetDatapoi
}
/**
- * Exports datapoints as CSV using SQL; the export path used in the SQL query must also be mapped
- * into the manager container so it can be accessed by this process. Backwards compatible overload
- * with default format.
+ * Streams datapoints as CSV using PostgreSQL COPY TO STDOUT. Backwards compatible overload with
+ * default format.
*/
public PipedInputStream exportDatapoints(
AttributeRef[] attributeRefs, long fromTimestamp, long toTimestamp) throws IOException {
@@ -292,8 +291,8 @@ public class AssetDatapointService extends AbstractDatapointService<AssetDatapoi
}
/**
- * Exports datapoints as CSV using SQL; the export path used in the SQL query must also be mapped
- * into the manager container so it can be accessed by this process.
+ * Streams datapoints as CSV using PostgreSQL COPY TO STDOUT through an in-memory pipe. No shared
+ * filesystem or intermediate export file is required.
*/
public PipedInputStream exportDatapoints(
AttributeRef[] attributeRefs,
diff --git a/profile/demo.yml b/profile/demo.yml
index 9b9007ce1..ad4ec092f 100644
--- a/profile/demo.yml
+++ b/profile/demo.yml
@@ -74,7 +74,6 @@ services:
- work_mem=4MB
volumes:
- postgresql-data:/var/lib/postgresql/data
- - manager-data:/storage
<<: *awslogs
keycloak:
diff --git a/profile/deploy.yml b/profile/deploy.yml
index f957808bb..80c700f76 100644
--- a/profile/deploy.yml
+++ b/profile/deploy.yml
@@ -449,10 +449,8 @@ services:
- -c
- work_mem=${OR_DB_WORK_MEM:-4MB}
# Docker volume for persistent storage of DB; can alternatively map to host
- volumes:
- # Needs to be available on the same path as the manager for data exports to work
- - manager-data:/storage
- # - postgresql-data:/var/lib/postgresql/data
+ #volumes:
+ # - postgresql-data:/var/lib/postgresql/data
#ports:
# - "5432:5432" # DB server
environment:
diff --git a/profile/dev-proxy.yml b/profile/dev-proxy.yml
index 4ed80c6ef..402a19f3d 100644
--- a/profile/dev-proxy.yml
+++ b/profile/dev-proxy.yml
@@ -34,7 +34,6 @@ services:
file: deploy.yml
service: postgresql
volumes:
- - ../tmp:/storage
- postgresql-data:/var/lib/postgresql/data
ports:
- "5432:5432"
diff --git a/profile/dev-testing.yml b/profile/dev-testing.yml
index ada6c954d..72f7b7520 100644
--- a/profile/dev-testing.yml
+++ b/profile/dev-testing.yml
@@ -22,7 +22,6 @@ services:
file: deploy.yml
service: postgresql
volumes:
- - ../tmp:/storage
- postgresql-data:/var/lib/postgresql/data
# Access directly if needed on localhost
ports:
diff --git a/profile/dev-ui.yml b/profile/dev-ui.yml
index c9c857af6..87a037bff 100644
--- a/profile/dev-ui.yml
+++ b/profile/dev-ui.yml
@@ -29,9 +29,7 @@ services:
extends:
file: deploy.yml
service: postgresql
- volumes:
- - manager-data:/storage
- - ../tmp/datapoint:/storage/datapoint
+ # No volume is mapped at all with this configuration, DB is ephemeral
# Access directly if needed on localhost
ports:
- "5432:5432"
diff --git a/test/load1/deployment1/test-load1.yml b/test/load1/deployment1/test-load1.yml
index 589ff4d76..7841b55a9 100644
--- a/test/load1/deployment1/test-load1.yml
+++ b/test/load1/deployment1/test-load1.yml
@@ -47,7 +47,6 @@ services:
POSTGRES_MAX_CONNECTIONS: ${POSTGRES_MAX_CONNECTIONS:-100}
volumes:
- postgresql-data:/var/lib/postgresql/data
- - manager-data:/storage
keycloak:
restart: always
diff --git a/test/src/test/groovy/org/openremote/test/assets/AssetDatapointExportTest.groovy b/test/src/test/groovy/org/openremote/test/assets/AssetDatapointExportTest.groovy
index f2516be4e..878a99813 100644
--- a/test/src/test/groovy/org/openremote/test/assets/AssetDatapointExportTest.groovy
+++ b/test/src/test/groovy/org/openremote/test/assets/AssetDatapointExportTest.groovy
@@ -108,7 +108,7 @@ class AssetDatapointExportTest extends Specification implements ManagerContainer
]
)
- then: "the default CSV export should return a file"
+ then: "the default CSV export should return a stream"
def inputStream1 = assetDatapointService.exportDatapoints(
[new AttributeRef(asset.id, attributeName)] as AttributeRef[],
dateTime.minusMinutes(30).atZone(ZoneId.systemDefault()).toInstant().toEpochMilli(),