Skip to content

Commit 12a455f

Browse files
hanshuaihanshuai
authored
Merge pull request #269 from sarike/ds-gcs
feat: google cloud storage
2 parents 9796063 + 6792e51 commit 12a455f

23 files changed

Lines changed: 802 additions & 137 deletions

File tree

‎server/node-service/package.json‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717
"jest": "^29.3.1",
1818
"nock": "^13.3.0",
1919
"nodemon": "^2.0.20",
20+
"svgo": "^3.0.2",
2021
"ts-jest": "^29.0.3",
2122
"ts-node": "^10.9.1"
2223
},
@@ -27,6 +28,7 @@
2728
"@aws-sdk/client-lambda": "^3.272.0",
2829
"@aws-sdk/client-s3": "^3.238.0",
2930
"@aws-sdk/s3-request-presigner": "^3.241.0",
31+
"@google-cloud/storage": "^6.9.3",
3032
"@types/axios": "^0.14.0",
3133
"@types/express": "^4.17.14",
3234
"@types/jsonpath": "^0.2.0",

‎server/node-service/src/controllers/plugins.ts‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,6 @@ export async function listPlugins(req: Request, res: Response) {
1111
}
1212
const ctx = pluginServices.getPluginContext(req);
1313
const result = pluginServices.listPlugins(ctx, ids as string[]);
14-
console.info("plugins: ", result);
1514
return res.status(200).json(result);
1615
}
1716

‎server/node-service/src/plugins/googleCloudStorage/dataSourceConfig.ts‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
import { ConfigToType } from "openblocks-sdk/dataSource";
2+
3+
export const dataSourceConfig = {
4+
type: "dataSource",
5+
params: [
6+
{
7+
key: "privateKey",
8+
label: "Private Key",
9+
type: "password",
10+
tooltip:
11+
"The private key associated with a Service Account with GCS privileges, [Documentation](https://cloud.google.com/iam/docs/service-accounts?hl=zh-cn) for service accounts.",
12+
rules: [
13+
{ required: true, message: "Please input your private key of google Service Account" },
14+
],
15+
},
16+
{
17+
key: "region",
18+
type: "textInput",
19+
label: "Region",
20+
},
21+
],
22+
} as const;
23+
24+
export type DataSourceDataType = ConfigToType<typeof dataSourceConfig>;

‎server/node-service/src/plugins/googleCloudStorage/index.ts‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
import { S3ServiceException } from "@aws-sdk/client-s3";
2+
import { ServiceError } from "../../common/error";
3+
import _ from "lodash";
4+
import { PluginContext } from "openblocks-sdk/dataSource";
5+
import queryConfig, { ActionDataType } from "./queryConfig";
6+
import { DataSourceDataType } from "./dataSourceConfig";
7+
import run, { validateDataSourceConfig } from "./run";
8+
import { dataSourceConfig } from "./dataSourceConfig";
9+
10+
const gcsPlugin = {
11+
id: "googleCloudStorage",
12+
name: "Google Cloud Storage",
13+
icon: "gcs.svg",
14+
category: "api",
15+
dataSourceConfig,
16+
queryConfig: queryConfig,
17+
18+
validateDataSourceConfig: async (dataSourceConfig: DataSourceDataType) => {
19+
return validateDataSourceConfig(dataSourceConfig);
20+
},
21+
22+
run: async (action: ActionDataType, dataSourceConfig: DataSourceDataType, ctx: PluginContext) => {
23+
try {
24+
return await run(action, dataSourceConfig);
25+
} catch (e) {
26+
if (e instanceof S3ServiceException) {
27+
throw new ServiceError(e.message, 400);
28+
}
29+
throw e;
30+
}
31+
},
32+
};
33+
34+
export default gcsPlugin;

‎server/node-service/src/plugins/googleCloudStorage/queryConfig.ts‎

Lines changed: 120 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,120 @@
1+
import { Config, ConfigToType, QueryConfig } from "openblocks-sdk/dataSource";
2+
3+
const bucketActionParam = {
4+
key: "bucket",
5+
type: "textInput",
6+
label: "Bucket",
7+
} as const;
8+
9+
const returnSignedUrlParam = {
10+
key: "returnSignedUrl",
11+
type: "switch",
12+
label: "Return signed url",
13+
placeholder: "false",
14+
} as const;
15+
16+
const queryConfig = {
17+
type: "query",
18+
label: "Action",
19+
actions: [
20+
{
21+
actionName: "listBuckets",
22+
label: "List Buckets",
23+
params: [],
24+
},
25+
{
26+
actionName: "listObjects",
27+
label: "List Files",
28+
params: [
29+
bucketActionParam,
30+
{
31+
key: "prefix",
32+
type: "textInput",
33+
label: "Prefix",
34+
},
35+
{
36+
key: "delimiter",
37+
type: "textInput",
38+
label: "Delimiter",
39+
},
40+
{
41+
key: "limit",
42+
type: "numberInput",
43+
defaultValue: 10,
44+
label: "Limit",
45+
},
46+
returnSignedUrlParam,
47+
],
48+
},
49+
{
50+
actionName: "readFile",
51+
label: "Read File",
52+
params: [
53+
bucketActionParam,
54+
{
55+
key: "fileName",
56+
type: "textInput",
57+
label: "File Name",
58+
},
59+
{
60+
key: "encoding",
61+
type: "select",
62+
label: "Data Type",
63+
options: [
64+
{ label: "Base64", value: "base64" },
65+
{ label: "Text", value: "utf8" },
66+
],
67+
},
68+
],
69+
},
70+
{
71+
actionName: "uploadData",
72+
label: "Upload File",
73+
params: [
74+
bucketActionParam,
75+
{
76+
key: "fileName",
77+
type: "textInput",
78+
label: "File Name",
79+
},
80+
{
81+
key: "encoding",
82+
type: "select",
83+
label: "Data Type",
84+
options: [
85+
{ label: "Base64", value: "base64" },
86+
{ label: "Text", value: "utf8" },
87+
],
88+
},
89+
{
90+
key: "data",
91+
type: "textInput",
92+
label: "Data",
93+
},
94+
{
95+
key: "contentType",
96+
type: "textInput",
97+
label: "Content-Type",
98+
placeholder: "image/png",
99+
},
100+
returnSignedUrlParam,
101+
],
102+
},
103+
{
104+
actionName: "deleteFile",
105+
label: "Delete File",
106+
params: [
107+
bucketActionParam,
108+
{
109+
key: "fileName",
110+
type: "textInput",
111+
label: "fileName",
112+
},
113+
],
114+
},
115+
],
116+
} as const;
117+
118+
export type ActionDataType = ConfigToType<typeof queryConfig>;
119+
120+
export default queryConfig;

‎server/node-service/src/plugins/googleCloudStorage/run.test.ts‎

Lines changed: 171 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,171 @@
1+
import getI18nTranslator from "./i18n";
2+
import run, { validateDataSourceConfig } from "./run";
3+
4+
const dataSourceConfig = {
5+
accessKey: "",
6+
secretKey: "",
7+
endpointUrl: "",
8+
region: "us-west-2",
9+
};
10+
11+
const bucket = "openblocks-demo";
12+
const i18n = getI18nTranslator(["en"]);
13+
14+
describe.skip("s3 plugin", () => {
15+
test("validate data source config", async () => {
16+
const a = await validateDataSourceConfig(dataSourceConfig);
17+
expect(a.success).toBe(true);
18+
19+
const b = await validateDataSourceConfig({
20+
...dataSourceConfig,
21+
accessKey: "error ak",
22+
});
23+
console.info(b.message);
24+
expect(b.success).toBe(false);
25+
});
26+
27+
test("read not existed file", async () => {
28+
await expect(
29+
run(
30+
{
31+
actionName: "readFile",
32+
bucket,
33+
fileName: "not-found.txt",
34+
encoding: "utf8",
35+
},
36+
dataSourceConfig,
37+
i18n
38+
)
39+
).rejects.toThrow();
40+
});
41+
42+
test("list buckets", async () => {
43+
const buckets = await run(
44+
{
45+
actionName: "listBuckets",
46+
},
47+
dataSourceConfig,
48+
i18n
49+
);
50+
expect(buckets?.length).toBeGreaterThan(0);
51+
});
52+
53+
test("crud file", async () => {
54+
const txtFileName = `ut-${Date.now()}.txt`;
55+
const binFileName = `ut-${Date.now()}.txt`;
56+
const fileData = `This is file ${txtFileName}`;
57+
const binRawFileData = `This is file ${binFileName}`;
58+
const binFileData = Buffer.from(binRawFileData).toString("base64");
59+
60+
// upload txt
61+
const uploadRes = await run(
62+
{
63+
actionName: "uploadData",
64+
fileName: txtFileName,
65+
encoding: "utf8",
66+
contentType: "text/plain",
67+
data: fileData,
68+
bucket,
69+
returnSignedUrl: true,
70+
},
71+
dataSourceConfig,
72+
i18n
73+
);
74+
expect((uploadRes as any).signedUrl).not.toBe("");
75+
76+
// upload bin
77+
const uploadBinRes = await run(
78+
{
79+
actionName: "uploadData",
80+
fileName: binFileName,
81+
encoding: "base64",
82+
contentType: "",
83+
data: binFileData,
84+
bucket,
85+
returnSignedUrl: true,
86+
},
87+
dataSourceConfig,
88+
i18n
89+
);
90+
expect((uploadBinRes as any).signedUrl).not.toBe("");
91+
92+
// list
93+
const list = await run(
94+
{
95+
actionName: "listObjects",
96+
bucket,
97+
prefix: txtFileName,
98+
delimiter: "",
99+
limit: 10,
100+
returnSignedUrl: false,
101+
},
102+
dataSourceConfig,
103+
i18n
104+
);
105+
expect((list as any).length).toBeGreaterThan(0);
106+
expect((list as any).find((i: any) => i.name === txtFileName)).not.toBeUndefined();
107+
expect((list as any).find((i: any) => i.name === binFileName)).not.toBeUndefined();
108+
109+
// read txt
110+
const readRes = await run(
111+
{
112+
actionName: "readFile",
113+
bucket,
114+
fileName: txtFileName,
115+
encoding: "utf8",
116+
},
117+
dataSourceConfig,
118+
i18n
119+
);
120+
expect((readRes as any).data).toEqual(fileData);
121+
122+
// read bin
123+
const readBinRes = await run(
124+
{
125+
actionName: "readFile",
126+
bucket,
127+
fileName: binFileName,
128+
encoding: "base64",
129+
},
130+
dataSourceConfig,
131+
i18n
132+
);
133+
expect((readBinRes as any).data).toEqual(binFileData);
134+
135+
// delete
136+
const delRes = await run(
137+
{
138+
actionName: "deleteFile",
139+
bucket,
140+
fileName: txtFileName,
141+
},
142+
dataSourceConfig,
143+
i18n
144+
);
145+
expect((delRes as any).success).toBe(true);
146+
147+
const delBinRes = await run(
148+
{
149+
actionName: "deleteFile",
150+
bucket,
151+
fileName: binFileName,
152+
},
153+
dataSourceConfig,
154+
i18n
155+
);
156+
expect((delBinRes as any).success).toBe(true);
157+
158+
// delete not empty file
159+
expect(
160+
run(
161+
{
162+
actionName: "deleteFile",
163+
bucket,
164+
fileName: "",
165+
},
166+
dataSourceConfig,
167+
i18n
168+
)
169+
).rejects.toThrow();
170+
});
171+
});

0 commit comments

Comments
 (0)