Skip to content

Commit 3aeb4e9

Browse files
lvhuichaolvhuichao
committed
1. add unit test and fix issues during testing
2. support GUI for postgres and multiple queries for UpdateOrDeleteSingleCommandResult 3. refactor MySQLQ GUI commands and implement Postgres GUI commands 4. fix connection pool parameter issue for Postgres 5. support js data source plugins
1 parent b447ac7 commit 3aeb4e9

110 files changed

Lines changed: 3200 additions & 706 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎deploy/docker/Dockerfile‎

Lines changed: 4 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -13,12 +13,6 @@ ARG REACT_APP_ENV=production
1313
ARG REACT_APP_EDITION=community
1414
RUN yarn build
1515

16-
FROM node:slim AS build-node
17-
COPY ./server/node-service /openblocks-node
18-
WORKDIR /openblocks-node
19-
RUN yarn --immutable
20-
RUN yarn build
21-
2216
FROM openjdk:17-slim AS jre-build
2317
RUN jlink --add-modules java.base,java.compiler,java.datatransfer,java.desktop,java.instrument,java.logging,java.management,java.management.rmi,java.naming,java.net.http,java.prefs,java.rmi,java.scripting,java.se,java.security.jgss,java.security.sasl,java.smartcardio,java.sql,java.sql.rowset,java.transaction.xa,java.xml,java.xml.crypto,jdk.accessibility,jdk.charsets,jdk.crypto.cryptoki,jdk.crypto.ec,jdk.dynalink,jdk.httpserver,jdk.incubator.foreign,jdk.incubator.vector,jdk.internal.vm.ci,jdk.jdwp.agent,jdk.jfr,jdk.jsobject,jdk.localedata,jdk.management,jdk.management.agent,jdk.management.jfr,jdk.naming.dns,jdk.naming.rmi,jdk.net,jdk.nio.mapmode,jdk.sctp,jdk.security.auth,jdk.security.jgss,jdk.unsupported,jdk.xml.dom,jdk.zipfs,jdk.attach \
2418
--output /build/jre \
@@ -45,12 +39,10 @@ RUN echo "deb [ arch=amd64,arm64 ]http://repo.mongodb.org/apt/ubuntu focal/mongo
4539
&& apt-get remove wget -y
4640

4741
RUN curl -sL https://deb.nodesource.com/setup_14.x | bash - \
48-
&& apt-get -y install --no-install-recommends -y mongodb-org=4.4.6 redis nodejs \
42+
&& apt-get -y install --no-install-recommends -y mongodb-org=4.4.6 redis \
4943
&& apt-get clean \
5044
&& rm -rf /var/lib/apt/lists/*
5145

52-
RUN npm install -g yarn
53-
5446
# Clean up cache file - Service layer
5547
RUN rm -rf \
5648
/root/.cache \
@@ -66,7 +58,6 @@ RUN rm -rf \
6658
VOLUME [ "/openblocks-stacks" ]
6759

6860
ENV OPENBLOCKS_SERVER_PROXY_PASS http://localhost:8080
69-
ENV OPENBLOCKS_NODE_PROXY_PASS http://localhost:6060
7061

7162
# copy jre
7263
COPY --from=jre-build /build/jre /app
@@ -76,16 +67,13 @@ ARG PLUGIN_JARS=/openblocks-server/openblocks-plugins/*/target/*.jar
7667

7768
RUN mkdir -p /openblocks /openblocks/plugins /var/www/openblocks /env2
7869

79-
COPY --from=build-server ${JAR_FILE} /openblocks/server/api-service/server.jar
80-
COPY --from=build-server ${PLUGIN_JARS} /openblocks/server/api-service/plugins/
70+
COPY --from=build-server ${JAR_FILE} /openblocks/server/server.jar
71+
COPY --from=build-server ${PLUGIN_JARS} /openblocks/server/plugins/
72+
8173

8274
# copy tace-fe build
8375
COPY --from=build-client /openblocks-client/packages/openblocks/build/ /openblocks/client/
8476

85-
# copy node-service
86-
COPY --from=build-node /openblocks-node /openblocks/server/node-service
87-
COPY ./deploy/docker/scripts/start-node-service.sh /openblocks/server/node-service/start-node-service.sh
88-
8977
# copy nginx conf
9078
COPY ./deploy/docker/templates/nginx/* /openblocks/nginx/
9179
COPY ./deploy/docker/scripts/start-nginx.sh /openblocks/nginx/start-nginx.sh

‎deploy/docker/entrypoint.sh‎

Lines changed: 11 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -2,15 +2,10 @@
22

33
set -e
44

5-
init_directory() {
6-
# Create sub-directory to store services log in the container mounting folder
7-
mkdir -p /openblocks-stacks/logs/backend
8-
mkdir -p /openblocks-stacks/logs/frontend
9-
mkdir -p /openblocks-stacks/logs/mongodb
10-
mkdir -p /openblocks-stacks/logs/redis
11-
mkdir -p /openblocks-stacks/logs/node
12-
mkdir -p /openblocks-stacks/data/redis
13-
mkdir -p /openblocks-stacks/data/mongodb
5+
init_mongodb() {
6+
echo "Init mongoDB"
7+
MONGO_DB_PATH="/openblocks-stacks/data/mongodb"
8+
mkdir -p "$MONGO_DB_PATH"
149
}
1510

1611
init_configuration() {
@@ -36,8 +31,14 @@ add_user() {
3631
chown $USER_ID:$GROUP_ID /run
3732
}
3833

39-
init_directory
34+
init_mongodb
4035
init_configuration
36+
# Create sub-directory to store services log in the container mounting folder
37+
mkdir -p /openblocks-stacks/logs/backend
38+
mkdir -p /openblocks-stacks/logs/frontend
39+
mkdir -p /openblocks-stacks/logs/redis
40+
mkdir -p /openblocks-stacks/data/redis
41+
4142
add_user
4243

4344
# Handle CMD command

‎deploy/docker/scripts/start-nginx.sh‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -9,9 +9,9 @@ cp /openblocks/nginx/nginx-root.conf.template /etc/nginx/nginx.conf
99

1010
if [ -e "/openblocks-stacks/ssl/fullchain.pem" ] && [ -e "/openblocks-stacks/ssl/privkey.pem" ];
1111
then
12-
envsubst '$OPENBLOCKS_SERVER_PROXY_PASS,$OPENBLOCKS_NODE_PROXY_PASS' < /openblocks/nginx/nginx-app-https.conf.template > /etc/nginx/conf.d/default.conf
12+
envsubst '$OPENBLOCKS_SERVER_PROXY_PASS' < /openblocks/nginx/nginx-app-https.conf.template > /etc/nginx/conf.d/default.conf
1313
else
14-
envsubst '$OPENBLOCKS_SERVER_PROXY_PASS,$OPENBLOCKS_NODE_PROXY_PASS' < /openblocks/nginx/nginx-app-http.conf.template > /etc/nginx/conf.d/default.conf
14+
envsubst '$OPENBLOCKS_SERVER_PROXY_PASS' < /openblocks/nginx/nginx-app-http.conf.template > /etc/nginx/conf.d/default.conf
1515
fi
1616

1717
exec nginx -g 'daemon off;'

‎deploy/docker/templates/nginx/nginx-app-http.conf.template‎

Lines changed: 10 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ server {
33
root /openblocks/client;
44
location / {
55
try_files $uri /index.html;
6+
67
if ($request_filename ~* .*.(html|htm)$) {
78
add_header Cache-Control no-cache;
89
}
@@ -12,9 +13,14 @@ server {
1213
proxy_set_header X-Forwarded-Host $host;
1314
proxy_pass ${OPENBLOCKS_SERVER_PROXY_PASS};
1415
}
15-
location /node-service/plugin-icons {
16-
proxy_set_header X-Forwarded-Proto $scheme;
17-
proxy_set_header X-Forwarded-Host $host;
18-
proxy_pass ${OPENBLOCKS_NODE_PROXY_PASS};
16+
location /oauth2 {
17+
proxy_set_header X-Forwarded-Proto $scheme;
18+
proxy_set_header X-Forwarded-Host $host;
19+
proxy_pass ${OPENBLOCKS_SERVER_PROXY_PASS};
20+
}
21+
location /login {
22+
proxy_set_header X-Forwarded-Proto $scheme;
23+
proxy_set_header X-Forwarded-Host $host;
24+
proxy_pass ${OPENBLOCKS_SERVER_PROXY_PASS};
1925
}
2026
}

‎deploy/docker/templates/nginx/nginx-app-https.conf.template‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ server {
66
root /openblocks/client;
77
location / {
88
try_files $uri /index.html;
9+
910
if ($request_filename ~* .*.(html|htm)$) {
1011
add_header Cache-Control no-cache;
1112
}
@@ -15,10 +16,15 @@ server {
1516
proxy_set_header X-Forwarded-Host $host;
1617
proxy_pass ${OPENBLOCKS_SERVER_PROXY_PASS};
1718
}
18-
location /node-service/plugin-icons {
19+
location /oauth2 {
20+
proxy_set_header X-Forwarded-Proto $scheme;
21+
proxy_set_header X-Forwarded-Host $host;
22+
proxy_pass ${OPENBLOCKS_SERVER_PROXY_PASS};
23+
}
24+
location /login {
1925
proxy_set_header X-Forwarded-Proto $scheme;
2026
proxy_set_header X-Forwarded-Host $host;
21-
proxy_pass ${OPENBLOCKS_NODE_PROXY_PASS};
27+
proxy_pass ${OPENBLOCKS_SERVER_PROXY_PASS};
2228
}
2329
}
2430

‎deploy/docker/templates/supervisord/backend.conf‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
[program:backend]
2-
directory=/openblocks/server/api-service
2+
directory=/openblocks/server
33
# Ref -Dlog4j2.formatMsgNoLookups=true https://spring.io/blog/2021/12/10/log4j2-vulnerability-and-spring-boot
44
command=/app/bin/java -Djava.security.egd="file:/dev/./urandom" -Dlog4j2.formatMsgNoLookups=true -Dspring.config.location="file:///openblocks/env/application.yml,file:///openblocks-stacks/configuration/application-selfhost.yml" -jar server.jar
55
priority=15

‎server/api-service/.gitignore‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
.DS_Store
2-
../.idea
2+
.idea
33
*.iml
44
.env
55
.vscode/*

‎server/api-service/openblocks-domain/src/main/java/com/openblocks/domain/datasource/model/Datasource.java‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,13 +9,17 @@
99
import java.util.Optional;
1010
import java.util.Set;
1111

12+
import javax.annotation.Nullable;
13+
1214
import org.apache.commons.lang3.ObjectUtils;
15+
import org.springframework.data.annotation.Transient;
1316

1417
import com.fasterxml.jackson.annotation.JsonIgnore;
1518
import com.fasterxml.jackson.annotation.JsonProperty;
1619
import com.openblocks.domain.plugin.DatasourceMetaInfoConstants;
1720
import com.openblocks.sdk.models.DatasourceConnectionConfig;
1821
import com.openblocks.sdk.models.HasIdAndAuditing;
22+
import com.openblocks.sdk.models.JsDatasourceConnectionConfig;
1923
import com.openblocks.sdk.plugin.graphql.GraphQLDatasourceConfig;
2024
import com.openblocks.sdk.plugin.openblocksapi.OpenblocksApiDatasourceConfig;
2125
import com.openblocks.sdk.plugin.restapi.RestApiDatasourceConfig;
@@ -68,6 +72,10 @@ public class Datasource extends HasIdAndAuditing {
6872
private String organizationId;
6973
private int creationSource;
7074
private DatasourceStatus datasourceStatus;
75+
// for js data source plugin
76+
@Nullable
77+
@Transient
78+
private Object pluginDefinition;
7179

7280
@JsonProperty(value = "datasourceConfig")
7381
private DatasourceConnectionConfig detailConfig;
@@ -76,6 +84,9 @@ public Datasource mergeWith(Datasource updatedDatasource) {
7684
setName(updatedDatasource.getName());
7785
Optional.of(getDetailConfig())
7886
.ifPresentOrElse(currentDetailConfig -> {
87+
if (updatedDatasource.getDetailConfig() instanceof JsDatasourceConnectionConfig jsDatasourceConnectionConfig) {
88+
jsDatasourceConnectionConfig.setType(updatedDatasource.getType());
89+
}
7990
DatasourceConnectionConfig updatedDetailConfig =
8091
currentDetailConfig.mergeWithUpdatedConfig(updatedDatasource.getDetailConfig());
8192
setDetailConfig(updatedDetailConfig);

‎server/api-service/openblocks-domain/src/main/java/com/openblocks/domain/datasource/repository/DatasourceRepository.java‎

Lines changed: 77 additions & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -10,10 +10,13 @@
1010
import com.openblocks.domain.datasource.model.DatasourceCreationSource;
1111
import com.openblocks.domain.datasource.model.DatasourceDO;
1212
import com.openblocks.domain.datasource.model.DatasourceStatus;
13+
import com.openblocks.domain.datasource.service.DatasourceService;
1314
import com.openblocks.domain.encryption.EncryptionService;
15+
import com.openblocks.domain.plugin.client.DatasourcePluginClient;
1416
import com.openblocks.domain.plugin.service.DatasourceMetaInfoService;
1517
import com.openblocks.infra.mongo.MongoUpsertHelper;
1618
import com.openblocks.sdk.models.DatasourceConnectionConfig;
19+
import com.openblocks.sdk.models.JsDatasourceConnectionConfig;
1720
import com.openblocks.sdk.util.JsonUtils;
1821

1922
import lombok.extern.slf4j.Slf4j;
@@ -40,25 +43,32 @@ public class DatasourceRepository {
4043
@Autowired
4144
private MongoUpsertHelper mongoUpsertHelper;
4245

46+
@Autowired
47+
private DatasourcePluginClient datasourcePluginClient;
48+
49+
@Autowired
50+
private DatasourceService datasourceService;
51+
4352
public Mono<Datasource> findById(String datasourceId) {
4453
return repository.findById(datasourceId)
45-
.map(this::convertToDomainObjectAndDecrypt);
54+
.flatMap(this::convertToDomainObjectAndDecrypt);
4655
}
4756

4857
public Mono<Datasource> findWorkspacePredefinedDatasourceByOrgIdAndType(String organizationId, String type) {
4958
return repository.findByOrganizationIdAndTypeAndCreationSource(organizationId, type,
5059
DatasourceCreationSource.LEGACY_WORKSPACE_PREDEFINED.getValue())
51-
.map(this::convertToDomainObjectAndDecrypt);
60+
.flatMap(this::convertToDomainObjectAndDecrypt);
5261
}
5362

5463
public Flux<Datasource> findAllByOrganizationId(String orgId) {
5564
return repository.findAllByOrganizationId(orgId)
56-
.map(this::convertToDomainObjectAndDecrypt);
65+
.flatMap(this::convertToDomainObjectAndDecrypt);
5766
}
5867

5968
public Mono<Datasource> save(Datasource datasource) {
60-
return repository.save(encryptDataAndConvertToDataObject(datasource))
61-
.map(this::convertToDomainObjectAndDecrypt);
69+
return encryptDataAndConvertToDataObject(datasource)
70+
.flatMap(repository::save)
71+
.flatMap(this::convertToDomainObjectAndDecrypt);
6272
}
6373

6474
public Mono<Boolean> markDatasourceAsDeleted(String datasourceId) {
@@ -72,47 +82,70 @@ public Mono<Long> countByOrganizationId(String orgId) {
7282
}
7383

7484
@SuppressWarnings("DuplicatedCode")
75-
private Datasource convertToDomainObjectAndDecrypt(DatasourceDO datasourceDO) {
76-
Datasource result = new Datasource();
77-
result.setName(datasourceDO.getName());
78-
result.setType(datasourceDO.getType());
79-
result.setOrganizationId(datasourceDO.getOrganizationId());
80-
result.setCreationSource(datasourceDO.getCreationSource());
81-
result.setDatasourceStatus(datasourceDO.getDatasourceStatus());
82-
result.setId(datasourceDO.getId());
83-
result.setCreatedAt(datasourceDO.getCreatedAt());
84-
result.setUpdatedAt(datasourceDO.getUpdatedAt());
85-
result.setCreatedBy(datasourceDO.getCreatedBy());
86-
result.setModifiedBy(datasourceDO.getModifiedBy());
87-
88-
try {
89-
DatasourceConnectionConfig detailConfig = datasourceMetaInfoService.resolveDetailConfig(datasourceDO.getDetailConfig(), result.getType());
90-
DatasourceConnectionConfig decryptedDetailConfig = detailConfig.doDecrypt(encryptionService::decryptString);
91-
result.setDetailConfig(decryptedDetailConfig);
92-
} catch (Exception e) {
93-
log.error("resolve detail config error.{},{}", result.getType(), JsonUtils.toJson(datasourceDO.getDetailConfig()), e);
94-
}
95-
return result;
85+
private Mono<Datasource> convertToDomainObjectAndDecrypt(DatasourceDO datasourceDO) {
86+
87+
Mono<Datasource> datasourceMono = Mono.fromSupplier(() -> {
88+
Datasource result = new Datasource();
89+
result.setName(datasourceDO.getName());
90+
result.setType(datasourceDO.getType());
91+
result.setOrganizationId(datasourceDO.getOrganizationId());
92+
result.setCreationSource(datasourceDO.getCreationSource());
93+
result.setDatasourceStatus(datasourceDO.getDatasourceStatus());
94+
result.setId(datasourceDO.getId());
95+
result.setCreatedAt(datasourceDO.getCreatedAt());
96+
result.setUpdatedAt(datasourceDO.getUpdatedAt());
97+
result.setCreatedBy(datasourceDO.getCreatedBy());
98+
result.setModifiedBy(datasourceDO.getModifiedBy());
99+
return result;
100+
})
101+
.cache();
102+
103+
return datasourceMono
104+
.doOnNext(datasource -> {
105+
if (datasourceMetaInfoService.isJsDatasourcePlugin(datasource.getType())) {
106+
JsDatasourceConnectionConfig jsDatasourceConnectionConfig = new JsDatasourceConnectionConfig();
107+
jsDatasourceConnectionConfig.putAll(datasourceDO.getDetailConfig());
108+
datasource.setDetailConfig(jsDatasourceConnectionConfig);
109+
} else {
110+
DatasourceConnectionConfig detailConfig =
111+
datasourceMetaInfoService.resolveDetailConfig(datasourceDO.getDetailConfig(), datasource.getType());
112+
datasource.setDetailConfig(detailConfig);
113+
}
114+
})
115+
.delayUntil(datasourceService::processJsDatasourcePlugin)
116+
.doOnNext(datasource -> {
117+
DatasourceConnectionConfig decryptedDetailConfig = datasource.getDetailConfig().doDecrypt(encryptionService::decryptString);
118+
// override
119+
datasource.setDetailConfig(decryptedDetailConfig);
120+
})
121+
.doOnError(throwable -> log.error("resolve detail config error.{},{}", datasourceDO.getType(),
122+
JsonUtils.toJson(datasourceDO.getDetailConfig()), throwable))
123+
.onErrorResume(__ -> datasourceMono);
96124
}
97125

98126
@SuppressWarnings("DuplicatedCode")
99-
private DatasourceDO encryptDataAndConvertToDataObject(Datasource datasource) {
100-
101-
DatasourceDO result = new DatasourceDO();
102-
result.setName(datasource.getName());
103-
result.setType(datasource.getType());
104-
result.setOrganizationId(datasource.getOrganizationId());
105-
result.setCreationSource(datasource.getCreationSource());
106-
result.setDatasourceStatus(datasource.getDatasourceStatus());
107-
result.setId(datasource.getId());
108-
result.setCreatedAt(datasource.getCreatedAt());
109-
result.setUpdatedAt(datasource.getUpdatedAt());
110-
result.setCreatedBy(datasource.getCreatedBy());
111-
result.setModifiedBy(datasource.getModifiedBy());
112-
113-
DatasourceConnectionConfig detailConfig = datasource.getDetailConfig();
114-
DatasourceConnectionConfig encryptedConfig = detailConfig.doEncrypt(encryptionService::encryptString);
115-
result.setDetailConfig(fromJsonMap(toJson(encryptedConfig)));
116-
return result;
127+
private Mono<DatasourceDO> encryptDataAndConvertToDataObject(Datasource datasource) {
128+
129+
return Mono.fromSupplier(() -> {
130+
DatasourceDO result = new DatasourceDO();
131+
result.setName(datasource.getName());
132+
result.setType(datasource.getType());
133+
result.setOrganizationId(datasource.getOrganizationId());
134+
result.setCreationSource(datasource.getCreationSource());
135+
result.setDatasourceStatus(datasource.getDatasourceStatus());
136+
result.setId(datasource.getId());
137+
result.setCreatedAt(datasource.getCreatedAt());
138+
result.setUpdatedAt(datasource.getUpdatedAt());
139+
result.setCreatedBy(datasource.getCreatedBy());
140+
result.setModifiedBy(datasource.getModifiedBy());
141+
return result;
142+
})
143+
.delayUntil(__ -> datasourceService.processJsDatasourcePlugin(datasource))
144+
.doOnNext(datasourceDO -> {
145+
DatasourceConnectionConfig detailConfig = datasource.getDetailConfig();
146+
DatasourceConnectionConfig encryptedConfig = detailConfig.doEncrypt(encryptionService::encryptString);
147+
// override
148+
datasourceDO.setDetailConfig(fromJsonMap(toJson(encryptedConfig)));
149+
});
117150
}
118151
}

‎server/api-service/openblocks-domain/src/main/java/com/openblocks/domain/datasource/service/DatasourceService.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,8 @@ public interface DatasourceService {
1818

1919
Mono<DatasourceTestResult> testDatasource(Datasource datasource);
2020

21+
Mono<Void> processJsDatasourcePlugin(Datasource datasource);
22+
2123
Flux<Datasource> getByOrgId(String orgId);
2224

2325
Mono<Long> countByOrganizationId(String orgId);

0 commit comments

Comments
 (0)