feat: openapi, n8n, dynamodb datasource

Author: sarike

server/node-service/README.md

@@ -7,12 +7,11 @@ Write data source plugins with TypeScript.
 ### Develop
 
 ```bash
-yarn dev
+yarn && yarn dev
 ```
 
 ### Production
 
 ```bash
-yarn build
-yarn start
+yarn && yarn build && yarn start
 ```

server/node-service/jest.config.js

@@ -3,4 +3,5 @@ module.exports = {
   preset: "ts-jest",
   testEnvironment: "node",
   testTimeout: 60000,
+  testPathIgnorePatterns: ["/node_modules/", "/build/"],
 };

server/node-service/package.json

@@ -7,35 +7,43 @@
   },
   "main": "src/server.ts",
   "scripts": {
-    "dev": "nodemon src/server.ts",
+    "dev": "nodemon --files src/server.ts",
     "start": "node ./build/src/server.js",
     "test": "jest",
     "build": "rm -rf build/ && tsc && cp -r src/static build/src/static"
   },
   "devDependencies": {
     "@types/jest": "^29.2.4",
     "jest": "^29.3.1",
+    "nock": "^13.3.0",
     "nodemon": "^2.0.20",
     "ts-jest": "^29.0.3",
     "ts-node": "^10.9.1"
   },
   "dependencies": {
+    "@apidevtools/swagger-parser": "^10.1.0",
+    "@aws-sdk/client-dynamodb": "^3.266.1",
     "@aws-sdk/client-s3": "^3.238.0",
     "@aws-sdk/s3-request-presigner": "^3.241.0",
     "@types/axios": "^0.14.0",
     "@types/express": "^4.17.14",
+    "@types/jsonpath": "^0.2.0",
     "@types/lodash": "^4.14.190",
     "@types/morgan": "^1.9.3",
     "@types/node": "^18.11.18",
     "axios": "^1.2.0",
+    "dynamodb-data-types": "^4.0.1",
     "express": "^4.18.2",
     "express-async-errors": "^3.1.1",
+    "jsonpath": "^1.1.1",
     "lodash": "^4.17.21",
     "loglevel": "^1.8.1",
     "morgan": "^1.10.0",
+    "openapi-types": "^12.1.0",
     "openblocks-core": "^0.0.5",
-    "openblocks-sdk": "^0.0.30",
+    "openblocks-sdk": "^0.0.34",
     "stylis": "^4.1.3",
+    "swagger-client": "^3.18.5",
     "typescript": "^4.9.3"
   },
   "packageManager": "yarn@3.3.1"

server/node-service/src/api/queryApi.ts

@@ -7,7 +7,7 @@ const DEFAULT_EXECUTE_ACTION_TIMEOUT_MS = 15000;
 
 type QueryExecuteRequest = {
   libraryQueryName: string;
-  params?: { key: string; value: string }[];
+  params?: { key: string; value: any }[];
 };
 
 export class QueryApi extends Api {

server/node-service/src/app-env.d.ts

@@ -0,0 +1,9 @@
+declare module "swagger-client" {
+  const SwaggerClient: {
+    helpers: {
+      opId(operation: OpenAPI.Operation, path: string, method: string): string;
+    };
+    execute: any;
+  };
+  export default SwaggerClient;
+}

server/node-service/src/common/error.ts

@@ -5,3 +5,7 @@ export class ServiceError extends Error {
     this.code = code;
   }
 }
+
+export function badRequest(message: string) {
+  return new ServiceError(message, 400);
+}

server/node-service/src/common/util.ts

@@ -1,3 +1,5 @@
+import { relaxedJSONToJSON } from "openblocks-core";
+
 export function toString(value: any): string {
   if (value === undefined || value === null) {
     return "";
@@ -37,3 +39,16 @@ export function toBoolean(value: any): boolean {
   }
   return !!value;
 }
+
+export function toJsonValue(value: any): any {
+  if (typeof value !== "string") {
+    return value;
+  }
+  try {
+    const json = relaxedJSONToJSON(value, true);
+    return JSON.parse(json);
+  } catch (e) {
+    console.info("invalid json input:", value);
+    return {};
+  }
+}

server/node-service/src/controllers/plugins.ts

@@ -1,10 +1,16 @@
+import { badRequest } from "../common/error";
 import { Request, Response } from "express";
 import _ from "lodash";
+import { Config } from "openblocks-sdk/dataSource";
 import * as pluginServices from "../services/plugin";
 
 export async function listPlugins(req: Request, res: Response) {
+  let ids = req.query["id"] || [];
+  if (typeof ids === "string") {
+    ids = [ids];
+  }
   const ctx = pluginServices.getPluginContext(req);
-  const result = pluginServices.listPlugins(ctx);
+  const result = pluginServices.listPlugins(ctx, ids as string[]);
   return res.status(200).json(result);
 }
 
@@ -31,3 +37,28 @@ export async function validatePluginDataSourceConfig(req: Request, res: Response
   );
   return res.status(200).json(result);
 }
+
+type GetDynamicDefReqBody = {
+  pluginName: string;
+  path: string;
+  dataSourceConfig: any;
+}[];
+
+export async function getDynamicDef(req: Request, res: Response) {
+  const ctx = pluginServices.getPluginContext(req);
+  if (!Array.isArray(req.body)) {
+    throw badRequest("request body is not a valid array");
+  }
+  const fields = req.body as GetDynamicDefReqBody;
+  const result: Config[] = [];
+  for (const item of fields) {
+    const def = await pluginServices.getDynamicConfigDef(
+      item.pluginName,
+      item.path,
+      item.dataSourceConfig,
+      ctx
+    );
+    result.push(def);
+  }
+  return res.status(200).json(result);
+}

server/node-service/src/controllers/runJavascript.ts

@@ -22,8 +22,17 @@ type BatchEvalResult = {
 };
 
 async function evalJs(req: EvalRequest, cookie?: string): Promise<EvalResult> {
-  const runQueryLibrary = (query: string) =>
-    QueryApi.executeQuery({ libraryQueryName: query }, cookie).then((r) => {
+  const runQueryLibrary = (query: string, params?: Record<string, any>) =>
+    QueryApi.executeQuery(
+      {
+        libraryQueryName: query,
+        params:
+          typeof params === "object" && params !== null
+            ? Object.entries(params).map(([key, value]) => ({ key, value }))
+            : undefined,
+      },
+      cookie
+    ).then((r) => {
       const data = r.data;
       if (data.success) {
         return Promise.resolve(data.data);
@@ -33,6 +42,7 @@ async function evalJs(req: EvalRequest, cookie?: string): Promise<EvalResult> {
   try {
     return { result: await evalFunc(req.jsCode || "", { ...req.context, runQueryLibrary }) };
   } catch (e) {
+    console.error("get error when eval js:", e);
     return { error: getErrorMessage(e) };
   }
 }

server/node-service/src/plugins/dynamodb/dataSourceConfig.ts

@@ -0,0 +1,35 @@
+import { ConfigToType } from "openblocks-sdk/dataSource";
+
+const dataSourceConfig = {
+  type: "dataSource",
+  params: [
+    {
+      key: "accessKey",
+      label: "Access key ID",
+      type: "textInput",
+      placeholder: "<Your Access key ID>",
+      rules: [{ required: true, message: "Please input the Access Key ID" }],
+    },
+    {
+      key: "secretKey",
+      label: "Secret key",
+      type: "password",
+      rules: [{ required: true, message: "Please input the Secret Key" }],
+    },
+    {
+      key: "endpointUrl",
+      label: "Endpoint URL",
+      type: "textInput",
+    },
+    {
+      key: "region",
+      type: "textInput",
+      label: "Region",
+      defaultValue: "us-west-1",
+    },
+  ],
+} as const;
+
+export default dataSourceConfig;
+
+export type DataSourceDataType = ConfigToType<typeof dataSourceConfig>;