Skip to content

Commit 167e00a

Browse files
neon-balconyneon-balcony
authored
Merge pull request #188 from lvhuichao/develop
2 parents b447ac7 + 3aeb4e9 commit 167e00a

110 files changed

Lines changed: 3200 additions & 706 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎deploy/docker/Dockerfile‎

Lines changed: 4 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -13,12 +13,6 @@ ARG REACT_APP_ENV=production
1313
ARG REACT_APP_EDITION=community
1414
RUN yarn build
1515

16-
FROM node:slim AS build-node
17-
COPY ./server/node-service /openblocks-node
18-
WORKDIR /openblocks-node
19-
RUN yarn --immutable
20-
RUN yarn build
21-
2216
FROM openjdk:17-slim AS jre-build
2317
RUN jlink --add-modules java.base,java.compiler,java.datatransfer,java.desktop,java.instrument,java.logging,java.management,java.management.rmi,java.naming,java.net.http,java.prefs,java.rmi,java.scripting,java.se,java.security.jgss,java.security.sasl,java.smartcardio,java.sql,java.sql.rowset,java.transaction.xa,java.xml,java.xml.crypto,jdk.accessibility,jdk.charsets,jdk.crypto.cryptoki,jdk.crypto.ec,jdk.dynalink,jdk.httpserver,jdk.incubator.foreign,jdk.incubator.vector,jdk.internal.vm.ci,jdk.jdwp.agent,jdk.jfr,jdk.jsobject,jdk.localedata,jdk.management,jdk.management.agent,jdk.management.jfr,jdk.naming.dns,jdk.naming.rmi,jdk.net,jdk.nio.mapmode,jdk.sctp,jdk.security.auth,jdk.security.jgss,jdk.unsupported,jdk.xml.dom,jdk.zipfs,jdk.attach \
2418
--output /build/jre \
@@ -45,12 +39,10 @@ RUN echo "deb [ arch=amd64,arm64 ]http://repo.mongodb.org/apt/ubuntu focal/mongo
4539
&& apt-get remove wget -y
4640

4741
RUN curl -sL https://deb.nodesource.com/setup_14.x | bash - \
48-
&& apt-get -y install --no-install-recommends -y mongodb-org=4.4.6 redis nodejs \
42+
&& apt-get -y install --no-install-recommends -y mongodb-org=4.4.6 redis \
4943
&& apt-get clean \
5044
&& rm -rf /var/lib/apt/lists/*
5145

52-
RUN npm install -g yarn
53-
5446
# Clean up cache file - Service layer
5547
RUN rm -rf \
5648
/root/.cache \
@@ -66,7 +58,6 @@ RUN rm -rf \
6658
VOLUME [ "/openblocks-stacks" ]
6759

6860
ENV OPENBLOCKS_SERVER_PROXY_PASS http://localhost:8080
69-
ENV OPENBLOCKS_NODE_PROXY_PASS http://localhost:6060
7061

7162
# copy jre
7263
COPY --from=jre-build /build/jre /app
@@ -76,16 +67,13 @@ ARG PLUGIN_JARS=/openblocks-server/openblocks-plugins/*/target/*.jar
7667

7768
RUN mkdir -p /openblocks /openblocks/plugins /var/www/openblocks /env2
7869

79-
COPY --from=build-server ${JAR_FILE} /openblocks/server/api-service/server.jar
80-
COPY --from=build-server ${PLUGIN_JARS} /openblocks/server/api-service/plugins/
70+
COPY --from=build-server ${JAR_FILE} /openblocks/server/server.jar
71+
COPY --from=build-server ${PLUGIN_JARS} /openblocks/server/plugins/
72+
8173

8274
# copy tace-fe build
8375
COPY --from=build-client /openblocks-client/packages/openblocks/build/ /openblocks/client/
8476

85-
# copy node-service
86-
COPY --from=build-node /openblocks-node /openblocks/server/node-service
87-
COPY ./deploy/docker/scripts/start-node-service.sh /openblocks/server/node-service/start-node-service.sh
88-
8977
# copy nginx conf
9078
COPY ./deploy/docker/templates/nginx/* /openblocks/nginx/
9179
COPY ./deploy/docker/scripts/start-nginx.sh /openblocks/nginx/start-nginx.sh

‎deploy/docker/entrypoint.sh‎

Lines changed: 11 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -2,15 +2,10 @@
22

33
set -e
44

5-
init_directory() {
6-
# Create sub-directory to store services log in the container mounting folder
7-
mkdir -p /openblocks-stacks/logs/backend
8-
mkdir -p /openblocks-stacks/logs/frontend
9-
mkdir -p /openblocks-stacks/logs/mongodb
10-
mkdir -p /openblocks-stacks/logs/redis
11-
mkdir -p /openblocks-stacks/logs/node
12-
mkdir -p /openblocks-stacks/data/redis
13-
mkdir -p /openblocks-stacks/data/mongodb
5+
init_mongodb() {
6+
echo "Init mongoDB"
7+
MONGO_DB_PATH="/openblocks-stacks/data/mongodb"
8+
mkdir -p "$MONGO_DB_PATH"
149
}
1510

1611
init_configuration() {
@@ -36,8 +31,14 @@ add_user() {
3631
chown $USER_ID:$GROUP_ID /run
3732
}
3833

39-
init_directory
34+
init_mongodb
4035
init_configuration
36+
# Create sub-directory to store services log in the container mounting folder
37+
mkdir -p /openblocks-stacks/logs/backend
38+
mkdir -p /openblocks-stacks/logs/frontend
39+
mkdir -p /openblocks-stacks/logs/redis
40+
mkdir -p /openblocks-stacks/data/redis
41+
4142
add_user
4243

4344
# Handle CMD command

‎deploy/docker/scripts/start-nginx.sh‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -9,9 +9,9 @@ cp /openblocks/nginx/nginx-root.conf.template /etc/nginx/nginx.conf
99

1010
if [ -e "/openblocks-stacks/ssl/fullchain.pem" ] && [ -e "/openblocks-stacks/ssl/privkey.pem" ];
1111
then
12-
envsubst '$OPENBLOCKS_SERVER_PROXY_PASS,$OPENBLOCKS_NODE_PROXY_PASS' < /openblocks/nginx/nginx-app-https.conf.template > /etc/nginx/conf.d/default.conf
12+
envsubst '$OPENBLOCKS_SERVER_PROXY_PASS' < /openblocks/nginx/nginx-app-https.conf.template > /etc/nginx/conf.d/default.conf
1313
else
14-
envsubst '$OPENBLOCKS_SERVER_PROXY_PASS,$OPENBLOCKS_NODE_PROXY_PASS' < /openblocks/nginx/nginx-app-http.conf.template > /etc/nginx/conf.d/default.conf
14+
envsubst '$OPENBLOCKS_SERVER_PROXY_PASS' < /openblocks/nginx/nginx-app-http.conf.template > /etc/nginx/conf.d/default.conf
1515
fi
1616

1717
exec nginx -g 'daemon off;'

‎deploy/docker/templates/nginx/nginx-app-http.conf.template‎

Lines changed: 10 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ server {
33
root /openblocks/client;
44
location / {
55
try_files $uri /index.html;
6+
67
if ($request_filename ~* .*.(html|htm)$) {
78
add_header Cache-Control no-cache;
89
}
@@ -12,9 +13,14 @@ server {
1213
proxy_set_header X-Forwarded-Host $host;
1314
proxy_pass ${OPENBLOCKS_SERVER_PROXY_PASS};
1415
}
15-
location /node-service/plugin-icons {
16-
proxy_set_header X-Forwarded-Proto $scheme;
17-
proxy_set_header X-Forwarded-Host $host;
18-
proxy_pass ${OPENBLOCKS_NODE_PROXY_PASS};
16+
location /oauth2 {
17+
proxy_set_header X-Forwarded-Proto $scheme;
18+
proxy_set_header X-Forwarded-Host $host;
19+
proxy_pass ${OPENBLOCKS_SERVER_PROXY_PASS};
20+
}
21+
location /login {
22+
proxy_set_header X-Forwarded-Proto $scheme;
23+
proxy_set_header X-Forwarded-Host $host;
24+
proxy_pass ${OPENBLOCKS_SERVER_PROXY_PASS};
1925
}
2026
}

‎deploy/docker/templates/nginx/nginx-app-https.conf.template‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ server {
66
root /openblocks/client;
77
location / {
88
try_files $uri /index.html;
9+
910
if ($request_filename ~* .*.(html|htm)$) {
1011
add_header Cache-Control no-cache;
1112
}
@@ -15,10 +16,15 @@ server {
1516
proxy_set_header X-Forwarded-Host $host;
1617
proxy_pass ${OPENBLOCKS_SERVER_PROXY_PASS};
1718
}
18-
location /node-service/plugin-icons {
19+
location /oauth2 {
20+
proxy_set_header X-Forwarded-Proto $scheme;
21+
proxy_set_header X-Forwarded-Host $host;
22+
proxy_pass ${OPENBLOCKS_SERVER_PROXY_PASS};
23+
}
24+
location /login {
1925
proxy_set_header X-Forwarded-Proto $scheme;
2026
proxy_set_header X-Forwarded-Host $host;
21-
proxy_pass ${OPENBLOCKS_NODE_PROXY_PASS};
27+
proxy_pass ${OPENBLOCKS_SERVER_PROXY_PASS};
2228
}
2329
}
2430

‎deploy/docker/templates/supervisord/backend.conf‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
[program:backend]
2-
directory=/openblocks/server/api-service
2+
directory=/openblocks/server
33
# Ref -Dlog4j2.formatMsgNoLookups=true https://spring.io/blog/2021/12/10/log4j2-vulnerability-and-spring-boot
44
command=/app/bin/java -Djava.security.egd="file:/dev/./urandom" -Dlog4j2.formatMsgNoLookups=true -Dspring.config.location="file:///openblocks/env/application.yml,file:///openblocks-stacks/configuration/application-selfhost.yml" -jar server.jar
55
priority=15

‎server/api-service/.gitignore‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
.DS_Store
2-
../.idea
2+
.idea
33
*.iml
44
.env
55
.vscode/*

‎server/api-service/openblocks-domain/src/main/java/com/openblocks/domain/datasource/model/Datasource.java‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,13 +9,17 @@
99
import java.util.Optional;
1010
import java.util.Set;
1111

12+
import javax.annotation.Nullable;
13+
1214
import org.apache.commons.lang3.ObjectUtils;
15+
import org.springframework.data.annotation.Transient;
1316

1417
import com.fasterxml.jackson.annotation.JsonIgnore;
1518
import com.fasterxml.jackson.annotation.JsonProperty;
1619
import com.openblocks.domain.plugin.DatasourceMetaInfoConstants;
1720
import com.openblocks.sdk.models.DatasourceConnectionConfig;
1821
import com.openblocks.sdk.models.HasIdAndAuditing;
22+
import com.openblocks.sdk.models.JsDatasourceConnectionConfig;
1923
import com.openblocks.sdk.plugin.graphql.GraphQLDatasourceConfig;
2024
import com.openblocks.sdk.plugin.openblocksapi.OpenblocksApiDatasourceConfig;
2125
import com.openblocks.sdk.plugin.restapi.RestApiDatasourceConfig;
@@ -68,6 +72,10 @@ public class Datasource extends HasIdAndAuditing {
6872
private String organizationId;
6973
private int creationSource;
7074
private DatasourceStatus datasourceStatus;
75+
// for js data source plugin
76+
@Nullable
77+
@Transient
78+
private Object pluginDefinition;
7179

7280
@JsonProperty(value = "datasourceConfig")
7381
private DatasourceConnectionConfig detailConfig;
@@ -76,6 +84,9 @@ public Datasource mergeWith(Datasource updatedDatasource) {
7684
setName(updatedDatasource.getName());
7785
Optional.of(getDetailConfig())
7886
.ifPresentOrElse(currentDetailConfig -> {
87+
if (updatedDatasource.getDetailConfig() instanceof JsDatasourceConnectionConfig jsDatasourceConnectionConfig) {
88+
jsDatasourceConnectionConfig.setType(updatedDatasource.getType());
89+
}
7990
DatasourceConnectionConfig updatedDetailConfig =
8091
currentDetailConfig.mergeWithUpdatedConfig(updatedDatasource.getDetailConfig());
8192
setDetailConfig(updatedDetailConfig);

‎server/api-service/openblocks-domain/src/main/java/com/openblocks/domain/datasource/repository/DatasourceRepository.java‎

Lines changed: 77 additions & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -10,10 +10,13 @@
1010
import com.openblocks.domain.datasource.model.DatasourceCreationSource;
1111
import com.openblocks.domain.datasource.model.DatasourceDO;
1212
import com.openblocks.domain.datasource.model.DatasourceStatus;
13+
import com.openblocks.domain.datasource.service.DatasourceService;
1314
import com.openblocks.domain.encryption.EncryptionService;
15+
import com.openblocks.domain.plugin.client.DatasourcePluginClient;
1416
import com.openblocks.domain.plugin.service.DatasourceMetaInfoService;
1517
import com.openblocks.infra.mongo.MongoUpsertHelper;
1618
import com.openblocks.sdk.models.DatasourceConnectionConfig;
19+
import com.openblocks.sdk.models.JsDatasourceConnectionConfig;
1720
import com.openblocks.sdk.util.JsonUtils;
1821

1922
import lombok.extern.slf4j.Slf4j;
@@ -40,25 +43,32 @@ public class DatasourceRepository {
4043
@Autowired
4144
private MongoUpsertHelper mongoUpsertHelper;
4245

46+
@Autowired
47+
private DatasourcePluginClient datasourcePluginClient;
48+
49+
@Autowired
50+
private DatasourceService datasourceService;
51+
4352
public Mono<Datasource> findById(String datasourceId) {
4453
return repository.findById(datasourceId)
45-
.map(this::convertToDomainObjectAndDecrypt);
54+
.flatMap(this::convertToDomainObjectAndDecrypt);
4655
}
4756

4857
public Mono<Datasource> findWorkspacePredefinedDatasourceByOrgIdAndType(String organizationId, String type) {
4958
return repository.findByOrganizationIdAndTypeAndCreationSource(organizationId, type,
5059
DatasourceCreationSource.LEGACY_WORKSPACE_PREDEFINED.getValue())
51-
.map(this::convertToDomainObjectAndDecrypt);
60+
.flatMap(this::convertToDomainObjectAndDecrypt);
5261
}
5362

5463
public Flux<Datasource> findAllByOrganizationId(String orgId) {
5564
return repository.findAllByOrganizationId(orgId)
56-
.map(this::convertToDomainObjectAndDecrypt);
65+
.flatMap(this::convertToDomainObjectAndDecrypt);
5766
}
5867

5968
public Mono<Datasource> save(Datasource datasource) {
60-
return repository.save(encryptDataAndConvertToDataObject(datasource))
61-
.map(this::convertToDomainObjectAndDecrypt);
69+
return encryptDataAndConvertToDataObject(datasource)
70+
.flatMap(repository::save)
71+
.flatMap(this::convertToDomainObjectAndDecrypt);
6272
}
6373

6474
public Mono<Boolean> markDatasourceAsDeleted(String datasourceId) {
@@ -72,47 +82,70 @@ public Mono<Long> countByOrganizationId(String orgId) {
7282
}
7383

7484
@SuppressWarnings("DuplicatedCode")
75-
private Datasource convertToDomainObjectAndDecrypt(DatasourceDO datasourceDO) {
76-
Datasource result = new Datasource();
77-
result.setName(datasourceDO.getName());
78-
result.setType(datasourceDO.getType());
79-
result.setOrganizationId(datasourceDO.getOrganizationId());
80-
result.setCreationSource(datasourceDO.getCreationSource());
81-
result.setDatasourceStatus(datasourceDO.getDatasourceStatus());
82-
result.setId(datasourceDO.getId());
83-
result.setCreatedAt(datasourceDO.getCreatedAt());
84-
result.setUpdatedAt(datasourceDO.getUpdatedAt());
85-
result.setCreatedBy(datasourceDO.getCreatedBy());
86-
result.setModifiedBy(datasourceDO.getModifiedBy());
87-
88-
try {
89-
DatasourceConnectionConfig detailConfig = datasourceMetaInfoService.resolveDetailConfig(datasourceDO.getDetailConfig(), result.getType());
90-
DatasourceConnectionConfig decryptedDetailConfig = detailConfig.doDecrypt(encryptionService::decryptString);
91-
result.setDetailConfig(decryptedDetailConfig);
92-
} catch (Exception e) {
93-
log.error("resolve detail config error.{},{}", result.getType(), JsonUtils.toJson(datasourceDO.getDetailConfig()), e);
94-
}
95-
return result;
85+
private Mono<Datasource> convertToDomainObjectAndDecrypt(DatasourceDO datasourceDO) {
86+
87+
Mono<Datasource> datasourceMono = Mono.fromSupplier(() -> {
88+
Datasource result = new Datasource();
89+
result.setName(datasourceDO.getName());
90+
result.setType(datasourceDO.getType());
91+
result.setOrganizationId(datasourceDO.getOrganizationId());
92+
result.setCreationSource(datasourceDO.getCreationSource());
93+
result.setDatasourceStatus(datasourceDO.getDatasourceStatus());
94+
result.setId(datasourceDO.getId());
95+
result.setCreatedAt(datasourceDO.getCreatedAt());
96+
result.setUpdatedAt(datasourceDO.getUpdatedAt());
97+
result.setCreatedBy(datasourceDO.getCreatedBy());
98+
result.setModifiedBy(datasourceDO.getModifiedBy());
99+
return result;
100+
})
101+
.cache();
102+
103+
return datasourceMono
104+
.doOnNext(datasource -> {
105+
if (datasourceMetaInfoService.isJsDatasourcePlugin(datasource.getType())) {
106+
JsDatasourceConnectionConfig jsDatasourceConnectionConfig = new JsDatasourceConnectionConfig();
107+
jsDatasourceConnectionConfig.putAll(datasourceDO.getDetailConfig());
108+
datasource.setDetailConfig(jsDatasourceConnectionConfig);
109+
} else {
110+
DatasourceConnectionConfig detailConfig =
111+
datasourceMetaInfoService.resolveDetailConfig(datasourceDO.getDetailConfig(), datasource.getType());
112+
datasource.setDetailConfig(detailConfig);
113+
}
114+
})
115+
.delayUntil(datasourceService::processJsDatasourcePlugin)
116+
.doOnNext(datasource -> {
117+
DatasourceConnectionConfig decryptedDetailConfig = datasource.getDetailConfig().doDecrypt(encryptionService::decryptString);
118+
// override
119+
datasource.setDetailConfig(decryptedDetailConfig);
120+
})
121+
.doOnError(throwable -> log.error("resolve detail config error.{},{}", datasourceDO.getType(),
122+
JsonUtils.toJson(datasourceDO.getDetailConfig()), throwable))
123+
.onErrorResume(__ -> datasourceMono);
96124
}
97125

98126
@SuppressWarnings("DuplicatedCode")
99-
private DatasourceDO encryptDataAndConvertToDataObject(Datasource datasource) {
100-
101-
DatasourceDO result = new DatasourceDO();
102-
result.setName(datasource.getName());
103-
result.setType(datasource.getType());
104-
result.setOrganizationId(datasource.getOrganizationId());
105-
result.setCreationSource(datasource.getCreationSource());
106-
result.setDatasourceStatus(datasource.getDatasourceStatus());
107-
result.setId(datasource.getId());
108-
result.setCreatedAt(datasource.getCreatedAt());
109-
result.setUpdatedAt(datasource.getUpdatedAt());
110-
result.setCreatedBy(datasource.getCreatedBy());
111-
result.setModifiedBy(datasource.getModifiedBy());
112-
113-
DatasourceConnectionConfig detailConfig = datasource.getDetailConfig();
114-
DatasourceConnectionConfig encryptedConfig = detailConfig.doEncrypt(encryptionService::encryptString);
115-
result.setDetailConfig(fromJsonMap(toJson(encryptedConfig)));
116-
return result;
127+
private Mono<DatasourceDO> encryptDataAndConvertToDataObject(Datasource datasource) {
128+
129+
return Mono.fromSupplier(() -> {
130+
DatasourceDO result = new DatasourceDO();
131+
result.setName(datasource.getName());
132+
result.setType(datasource.getType());
133+
result.setOrganizationId(datasource.getOrganizationId());
134+
result.setCreationSource(datasource.getCreationSource());
135+
result.setDatasourceStatus(datasource.getDatasourceStatus());
136+
result.setId(datasource.getId());
137+
result.setCreatedAt(datasource.getCreatedAt());
138+
result.setUpdatedAt(datasource.getUpdatedAt());
139+
result.setCreatedBy(datasource.getCreatedBy());
140+
result.setModifiedBy(datasource.getModifiedBy());
141+
return result;
142+
})
143+
.delayUntil(__ -> datasourceService.processJsDatasourcePlugin(datasource))
144+
.doOnNext(datasourceDO -> {
145+
DatasourceConnectionConfig detailConfig = datasource.getDetailConfig();
146+
DatasourceConnectionConfig encryptedConfig = detailConfig.doEncrypt(encryptionService::encryptString);
147+
// override
148+
datasourceDO.setDetailConfig(fromJsonMap(toJson(encryptedConfig)));
149+
});
117150
}
118151
}

‎server/api-service/openblocks-domain/src/main/java/com/openblocks/domain/datasource/service/DatasourceService.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,8 @@ public interface DatasourceService {
1818

1919
Mono<DatasourceTestResult> testDatasource(Datasource datasource);
2020

21+
Mono<Void> processJsDatasourcePlugin(Datasource datasource);
22+
2123
Flux<Datasource> getByOrgId(String orgId);
2224

2325
Mono<Long> countByOrganizationId(String orgId);

0 commit comments

Comments
 (0)