Skip to content

Commit 951eecd

Browse files
simplestephsimplesteph
committed
first working version
1 parent e44d6a6 commit 951eecd

29 files changed

Lines changed: 2847 additions & 1 deletion

‎.gitignore‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
standalone.offsets
22
target/
33
.idea/
4-
*.log
4+
*.log
5+
*.iml

‎README.md‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
Welcome to your new Kafka Connect connector!
2+
3+
# Running in development
4+
5+
```
6+
mvn clean package
7+
export CLASSPATH="$(find target/ -type f -name '*.jar'| grep '\-package' | tr '\n' ':')"
8+
connect-standalone config/worker.properties config/GitHubSourceConnectorExample.properties
9+
```

‎build.sh‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
#!/usr/bin/env bash
2+
mvn clean package

‎config/GitHubSourceConnectorExample.properties‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
name=GitHubSourceConnectorDemo
2+
tasks.max=1
3+
connector.class=com.simplesteph.kafka.GitHubSourceConnector
4+
topic=github-issues
5+
github.owner=kubernetes
6+
github.repo=kubernetes
7+
since.timestamp=2017-01-01T00:00:00Z
8+
# I heavily recommend you set those two fields:
9+
# auth.username=your_username
10+
# auth.password=your_password

‎config/worker.properties‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
# from more information, visit: http://docs.confluent.io/3.2.0/connect/userguide.html#common-worker-configs
2+
bootstrap.servers=127.0.0.1:9092
3+
key.converter=org.apache.kafka.connect.json.JsonConverter
4+
key.converter.schemas.enable=true
5+
value.converter=org.apache.kafka.connect.json.JsonConverter
6+
value.converter.schemas.enable=true
7+
# we always leave the internal key to JsonConverter
8+
internal.key.converter=org.apache.kafka.connect.json.JsonConverter
9+
internal.key.converter.schemas.enable=true
10+
internal.value.converter=org.apache.kafka.connect.json.JsonConverter
11+
internal.value.converter.schemas.enable=true
12+
# Rest API
13+
rest.port=8086
14+
rest.host.name=127.0.0.1
15+
# this config is only for standalone workers
16+
offset.storage.file.filename=standalone.offsets
17+
offset.flush.interval.ms=10000

‎pom.xml‎

Lines changed: 107 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,107 @@
1+
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
2+
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
3+
<modelVersion>4.0.0</modelVersion>
4+
5+
<groupId>com.simplesteph.kafka</groupId>
6+
<artifactId>kafka-connect-github-source</artifactId>
7+
<version>1.0</version>
8+
<packaging>jar</packaging>
9+
10+
<name>kafka-connect-github-source</name>
11+
12+
<properties>
13+
<kafka.version>0.10.2.0-cp1</kafka.version>
14+
<junit.version>4.12</junit.version>
15+
</properties>
16+
17+
<dependencies>
18+
<dependency>
19+
<groupId>org.apache.kafka</groupId>
20+
<artifactId>connect-api</artifactId>
21+
<version>${kafka.version}</version>
22+
<scope>provided</scope>
23+
</dependency>
24+
<dependency>
25+
<groupId>junit</groupId>
26+
<artifactId>junit</artifactId>
27+
<version>${junit.version}</version>
28+
<scope>test</scope>
29+
</dependency>
30+
<dependency>
31+
<groupId>commons-io</groupId>
32+
<artifactId>commons-io</artifactId>
33+
<version>2.4</version>
34+
<scope>test</scope>
35+
</dependency>
36+
<!-- https://mvnrepository.com/artifact/org.slf4j/slf4j-log4j12 -->
37+
<dependency>
38+
<groupId>org.slf4j</groupId>
39+
<artifactId>slf4j-log4j12</artifactId>
40+
<version>1.7.25</version>
41+
</dependency>
42+
<!-- https://mvnrepository.com/artifact/com.mashape.unirest/unirest-java -->
43+
<dependency>
44+
<groupId>com.mashape.unirest</groupId>
45+
<artifactId>unirest-java</artifactId>
46+
<version>1.4.9</version>
47+
</dependency>
48+
49+
</dependencies>
50+
51+
<build>
52+
<plugins>
53+
<plugin>
54+
<groupId>org.apache.maven.plugins</groupId>
55+
<artifactId>maven-jar-plugin</artifactId>
56+
<configuration>
57+
<archive>
58+
<manifest>
59+
<addDefaultImplementationEntries>true</addDefaultImplementationEntries>
60+
<addDefaultSpecificationEntries>true</addDefaultSpecificationEntries>
61+
</manifest>
62+
</archive>
63+
</configuration>
64+
</plugin>
65+
<plugin>
66+
<groupId>org.apache.maven.plugins</groupId>
67+
<artifactId>maven-compiler-plugin</artifactId>
68+
<version>2.5.1</version>
69+
<inherited>true</inherited>
70+
<configuration>
71+
<source>1.8</source>
72+
<target>1.8</target>
73+
</configuration>
74+
</plugin>
75+
<plugin>
76+
<artifactId>maven-assembly-plugin</artifactId>
77+
<version>2.5.3</version>
78+
<configuration>
79+
<descriptors>
80+
<descriptor>src/main/assembly/package.xml</descriptor>
81+
</descriptors>
82+
</configuration>
83+
<executions>
84+
<execution>
85+
<id>make-assembly</id>
86+
<phase>package</phase>
87+
<goals>
88+
<goal>single</goal>
89+
</goals>
90+
</execution>
91+
</executions>
92+
</plugin>
93+
</plugins>
94+
<resources>
95+
<resource>
96+
<directory>src/main/resources</directory>
97+
<filtering>true</filtering>
98+
</resource>
99+
</resources>
100+
</build>
101+
<repositories>
102+
<repository>
103+
<id>confluent</id>
104+
<url>http://packages.confluent.io/maven/</url>
105+
</repository>
106+
</repositories>
107+
</project>

‎run.sh‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
#!/usr/bin/env bash
2+
export CLASSPATH="$(find target/ -type f -name '*.jar'| grep '\-package' | tr '\n' ':')"
3+
connect-standalone config/worker.properties config/GitHubSourceConnectorExample.properties

‎src/main/assembly/package.xml‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2"
2+
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
3+
xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2
4+
http://maven.apache.org/xsd/assembly-1.1.2.xsd">
5+
<!-- Assembles a packaged version targeting OS installation. -->
6+
<id>package</id>
7+
<formats>
8+
<format>dir</format>
9+
</formats>
10+
<includeBaseDirectory>false</includeBaseDirectory>
11+
<fileSets>
12+
<fileSet>
13+
<directory>${project.basedir}</directory>
14+
<outputDirectory>share/doc/${project.name}/</outputDirectory>
15+
<includes>
16+
<include>README*</include>
17+
<include>LICENSE*</include>
18+
<include>NOTICE*</include>
19+
<include>licenses/</include>
20+
</includes>
21+
</fileSet>
22+
<fileSet>
23+
<directory>${project.basedir}/config</directory>
24+
<outputDirectory>etc/${project.name}</outputDirectory>
25+
<includes>
26+
<include>*</include>
27+
</includes>
28+
</fileSet>
29+
</fileSets>
30+
<dependencySets>
31+
<dependencySet>
32+
<outputDirectory>share/java/${project.name}</outputDirectory>
33+
<useProjectArtifact>true</useProjectArtifact>
34+
<useTransitiveFiltering>true</useTransitiveFiltering>
35+
<excludes>
36+
<exclude>org.apache.kafka:connect-api</exclude>
37+
</excludes>
38+
</dependencySet>
39+
</dependencySets>
40+
</assembly>

‎src/main/java/com/simplesteph/kafka/GitHubSchemas.java‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
package com.simplesteph.kafka;
2+
3+
import org.apache.kafka.connect.data.Schema;
4+
import org.apache.kafka.connect.data.SchemaBuilder;
5+
6+
public class GitHubSchemas {
7+
8+
public static String NEXT_PAGE_FIELD = "next_page";
9+
10+
// Issue fields
11+
public static String FULL_REPO_FIELD = "owner/repository";
12+
public static String OWNER_FIELD = "owner";
13+
public static String REPOSITORY_FIELD = "repository";
14+
public static String CREATED_AT_FIELD = "created_at";
15+
public static String UPDATED_AT_FIELD = "updated_at";
16+
public static String NUMBER_FIELD = "number";
17+
public static String URL_FIELD = "url";
18+
public static String HTML_URL_FIELD = "html_url";
19+
public static String TITLE_FIELD = "title";
20+
public static String STATE_FIELD = "state";
21+
22+
// User fields
23+
public static String USER_FIELD = "user";
24+
public static String USER_URL_FIELD = "url";
25+
public static String USER_HTML_URL_FIELD = "html_url";
26+
public static String USER_ID_FIELD = "id";
27+
public static String USER_LOGIN_FIELD = "login";
28+
29+
// Schema names
30+
public static String SCHEMA_KEY = "GitHub Issue Key";
31+
public static String SCHEMA_VALUE_ISSUE = "GitHub Issue";
32+
public static String SCHEMA_VALUE_USER = "User";
33+
34+
// Key Schema
35+
public static Schema KEY_SCHEMA = SchemaBuilder.struct().name(SCHEMA_KEY)
36+
.version(1)
37+
.field(OWNER_FIELD, Schema.STRING_SCHEMA)
38+
.field(REPOSITORY_FIELD, Schema.STRING_SCHEMA)
39+
.field(NUMBER_FIELD, Schema.INT32_SCHEMA)
40+
.build();
41+
42+
// Value Schema
43+
public static Schema USER_SCHEMA = SchemaBuilder.struct().name(SCHEMA_VALUE_USER)
44+
.version(1)
45+
.field(USER_URL_FIELD, Schema.STRING_SCHEMA)
46+
.field(USER_ID_FIELD, Schema.INT32_SCHEMA)
47+
.field(USER_LOGIN_FIELD, Schema.STRING_SCHEMA)
48+
.build();
49+
50+
public static Schema VALUE_SCHEMA = SchemaBuilder.struct().name(SCHEMA_VALUE_ISSUE)
51+
.version(1)
52+
.field(URL_FIELD, Schema.STRING_SCHEMA)
53+
.field(TITLE_FIELD, Schema.STRING_SCHEMA)
54+
.field(CREATED_AT_FIELD, Schema.INT64_SCHEMA)
55+
.field(UPDATED_AT_FIELD, Schema.INT64_SCHEMA)
56+
.field(NUMBER_FIELD, Schema.INT32_SCHEMA)
57+
.field(STATE_FIELD, Schema.STRING_SCHEMA)
58+
.field(USER_FIELD, USER_SCHEMA)
59+
.build();
60+
}

‎src/main/java/com/simplesteph/kafka/GitHubSourceConnector.java‎

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
package com.simplesteph.kafka;
2+
3+
import java.util.ArrayList;
4+
import java.util.List;
5+
import java.util.Map;
6+
7+
import org.apache.kafka.common.config.ConfigDef;
8+
import org.apache.kafka.connect.connector.Task;
9+
import org.apache.kafka.connect.source.SourceConnector;
10+
import org.slf4j.Logger;
11+
import org.slf4j.LoggerFactory;
12+
13+
public class GitHubSourceConnector extends SourceConnector {
14+
private static Logger log = LoggerFactory.getLogger(GitHubSourceConnector.class);
15+
private GitHubSourceConnectorConfig config;
16+
17+
@Override
18+
public String version() {
19+
return VersionUtil.getVersion();
20+
}
21+
22+
@Override
23+
public void start(Map<String, String> map) {
24+
config = new GitHubSourceConnectorConfig(map);
25+
}
26+
27+
@Override
28+
public Class<? extends Task> taskClass() {
29+
return GitHubSourceTask.class;
30+
}
31+
32+
@Override
33+
public List<Map<String, String>> taskConfigs(int i) {
34+
// Define the individual task configurations that will be executed.
35+
ArrayList<Map<String, String>> configs = new ArrayList<>(1);
36+
configs.add(config.originalsStrings());
37+
return configs;
38+
}
39+
40+
@Override
41+
public void stop() {
42+
// Do things that are necessary to stop your connector.
43+
// nothing is necessary to stop for this connector
44+
}
45+
46+
@Override
47+
public ConfigDef config() {
48+
return GitHubSourceConnectorConfig.conf();
49+
}
50+
}

0 commit comments

Comments
 (0)