Skip to content

Commit 52b9d9d

Browse files
ambudambud
authored
Merge pull request #102 from srotya/development
Feature enhancements, new release
2 parents 4a8050e + 15f7d4a commit 52b9d9d

108 files changed

Lines changed: 5003 additions & 1208 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.travis.yml‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,8 +8,9 @@ addons:
88
- rpm
99
- wget
1010
- unzip
11+
- npm
1112
before_script:
12-
- DIR=$(pwd);mkdir /tmp/proto;cd /tmp/proto;wget https://github.com/google/protobuf/releases/download/v3.2.0/protoc-3.2.0-linux-x86_64.zip;unzip protoc-3.2.0-linux-x86_64.zip; ls -lh;cd $DIR
13+
- npm -g install polymer-cli;DIR=$(pwd);mkdir /tmp/proto;cd /tmp/proto;wget https://github.com/google/protobuf/releases/download/v3.2.0/protoc-3.2.0-linux-x86_64.zip;unzip protoc-3.2.0-linux-x86_64.zip; ls -lh;cd $DIR
1314
script: mvn -q -T2 clean test javadoc:javadoc
1415
after_success:
1516
- bash <(curl -s https://codecov.io/bash)

‎archiver/hdfs-archiver/src/main/java/com/srotya/sidewinder/archiver/hdfs/HDFSArchiver.java‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,6 @@
3636

3737
import com.srotya.sidewinder.core.storage.ArchiveException;
3838
import com.srotya.sidewinder.core.storage.Archiver;
39-
import com.srotya.sidewinder.core.storage.mem.archival.DiskArchiver;
4039
import com.srotya.sidewinder.core.storage.mem.archival.TimeSeriesArchivalObject;
4140

4241

@@ -92,7 +91,7 @@ public void archive(TimeSeriesArchivalObject object) throws ArchiveException {
9291
currentFile = new Path(archiveDirectory, "archive." + System.currentTimeMillis() + ".bin");
9392
os = new DataOutputStream(new BufferedOutputStream(FileSystem.create(fs, currentFile, fsPermission)));
9493
}
95-
DiskArchiver.serializeToStream(os, new TimeSeriesArchivalObject(object.getDb(), object.getMeasurement(),
94+
Archiver.serializeToStream(os, new TimeSeriesArchivalObject(object.getDb(), object.getMeasurement(),
9695
object.getKey(), object.getBucket()));
9796
} catch (IOException e) {
9897
throw new ArchiveException(e);
@@ -107,7 +106,7 @@ public List<TimeSeriesArchivalObject> unarchive() throws ArchiveException {
107106
while (itr.hasNext()) {
108107
DataInputStream bis = new DataInputStream(new BufferedInputStream(fs.open(itr.next().getPath())));
109108
while (bis.available() > 0) {
110-
TimeSeriesArchivalObject object = DiskArchiver.deserializeFromStream(bis);
109+
TimeSeriesArchivalObject object = Archiver.deserializeFromStream(bis);
111110
list.add(object);
112111
}
113112
bis.close();

‎archiver/s3-archiver/src/main/java/com/srotya/sidewinder/archiver/s3/S3Archiver.java‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,6 @@
3838
import com.amazonaws.services.s3.model.S3ObjectSummary;
3939
import com.srotya.sidewinder.core.storage.ArchiveException;
4040
import com.srotya.sidewinder.core.storage.Archiver;
41-
import com.srotya.sidewinder.core.storage.mem.archival.DiskArchiver;
4241
import com.srotya.sidewinder.core.storage.mem.archival.TimeSeriesArchivalObject;
4342

4443
/**
@@ -76,7 +75,7 @@ public void archive(TimeSeriesArchivalObject object) throws ArchiveException {
7675
try {
7776
ByteArrayOutputStream bytes = new ByteArrayOutputStream();
7877
DataOutputStream os = new DataOutputStream(bytes);
79-
DiskArchiver.serializeToStream(os, new TimeSeriesArchivalObject(object.getDb(), object.getMeasurement(),
78+
Archiver.serializeToStream(os, new TimeSeriesArchivalObject(object.getDb(), object.getMeasurement(),
8079
object.getKey(), object.getBucket()));
8180
os.close();
8281
bytes.close();
@@ -98,7 +97,7 @@ public List<TimeSeriesArchivalObject> unarchive() throws ArchiveException {
9897
S3Object object = s3.getObject(bucketName, summary.getKey());
9998
DataInputStream dis = new DataInputStream(object.getObjectContent());
10099
try {
101-
list.add(DiskArchiver.deserializeFromStream(dis));
100+
list.add(Archiver.deserializeFromStream(dis));
102101
} catch (IOException e) {
103102
throw new ArchiveException(e);
104103
}

‎core/pom.xml‎

Lines changed: 54 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -13,15 +13,9 @@
1313
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
1414
</properties>
1515
<dependencies>
16-
<dependency>
17-
<groupId>com.google.code.gson</groupId>
18-
<artifactId>gson</artifactId>
19-
<version>2.8.0</version>
20-
</dependency>
2116
<dependency>
2217
<groupId>com.google.guava</groupId>
2318
<artifactId>guava</artifactId>
24-
<version>19.0</version>
2519
</dependency>
2620
<dependency>
2721
<groupId>com.google.protobuf</groupId>
@@ -91,10 +85,22 @@
9185
<dependency>
9286
<groupId>io.grpc</groupId>
9387
<artifactId>grpc-all</artifactId>
88+
<exclusions>
89+
<exclusion>
90+
<groupId>com.google.guava</groupId>
91+
<artifactId>guava</artifactId>
92+
</exclusion>
93+
</exclusions>
9494
</dependency>
9595
<dependency>
9696
<groupId>io.grpc</groupId>
9797
<artifactId>grpc-protobuf</artifactId>
98+
<exclusions>
99+
<exclusion>
100+
<groupId>com.google.guava</groupId>
101+
<artifactId>guava</artifactId>
102+
</exclusion>
103+
</exclusions>
98104
</dependency>
99105
<dependency>
100106
<groupId>io.grpc</groupId>
@@ -105,12 +111,6 @@
105111
<artifactId>junit</artifactId>
106112
<scope>test</scope>
107113
</dependency>
108-
<dependency>
109-
<groupId>org.antlr</groupId>
110-
<artifactId>antlr4</artifactId>
111-
<version>4.6</version>
112-
<classifier>complete</classifier>
113-
</dependency>
114114
<dependency>
115115
<groupId>org.apache.calcite</groupId>
116116
<artifactId>calcite-core</artifactId>
@@ -128,8 +128,17 @@
128128
<groupId>com.fasterxml.jackson.core</groupId>
129129
<artifactId>jackson-annotations</artifactId>
130130
</exclusion>
131+
<exclusion>
132+
<groupId>com.google.guava</groupId>
133+
<artifactId>guava</artifactId>
134+
</exclusion>
131135
</exclusions>
132136
</dependency>
137+
<dependency>
138+
<groupId>org.apache.commons</groupId>
139+
<artifactId>commons-compress</artifactId>
140+
<version>1.8.1</version>
141+
</dependency>
133142
<dependency>
134143
<groupId>org.apache.commons</groupId>
135144
<artifactId>commons-math</artifactId>
@@ -150,13 +159,36 @@
150159
<groupId>org.mapdb</groupId>
151160
<artifactId>mapdb</artifactId>
152161
<version>3.0.5</version>
162+
<exclusions>
163+
<exclusion>
164+
<groupId>com.google.guava</groupId>
165+
<artifactId>guava</artifactId>
166+
</exclusion>
167+
</exclusions>
153168
</dependency>
154169
<dependency>
155170
<groupId>org.mockito</groupId>
156171
<artifactId>mockito-all</artifactId>
172+
<scope>test</scope>
173+
</dependency>
174+
<dependency>
175+
<groupId>org.nd4j</groupId>
176+
<artifactId>nd4j-api</artifactId>
177+
<version>0.9.1</version>
178+
</dependency>
179+
<dependency>
180+
<groupId>org.reflections</groupId>
181+
<artifactId>reflections</artifactId>
182+
<version>0.9.11</version>
157183
</dependency>
158184
</dependencies>
159185
<build>
186+
<resources>
187+
<resource>
188+
<targetPath>web</targetPath>
189+
<directory>target/build/default/</directory>
190+
</resource>
191+
</resources>
160192
<plugins>
161193
<plugin>
162194
<groupId>org.apache.maven.plugins</groupId>
@@ -199,20 +231,17 @@
199231
</executions>
200232
</plugin>
201233
<plugin>
202-
<groupId>org.xolstice.maven.plugins</groupId>
203-
<artifactId>protobuf-maven-plugin</artifactId>
204-
<version>0.5.0</version>
205-
<configuration>
206-
<protocArtifact>com.google.protobuf:protoc:3.0.2:exe:${os.detected.classifier}</protocArtifact>
207-
<pluginId>grpc-java</pluginId>
208-
<pluginArtifact>io.grpc:protoc-gen-grpc-java:1.2.0:exe:${os.detected.classifier}</pluginArtifact>
209-
</configuration>
234+
<groupId>org.codehaus.mojo</groupId>
235+
<artifactId>exec-maven-plugin</artifactId>
210236
<executions>
211237
<execution>
212-
<goals>
213-
<goal>compile</goal>
214-
<goal>compile-custom</goal>
215-
</goals>
238+
<!-- Run our version calculation script -->
239+
<id>Version Calculation</id>
240+
<goals />
241+
<phase>generate-sources</phase>
242+
<configuration>
243+
<executable>${basedir}/src/web/build.sh</executable>
244+
</configuration>
216245
</execution>
217246
</executions>
218247
</plugin>
@@ -287,6 +316,7 @@
287316
<script>mkdir -p /usr/sidewinder/lib</script>
288317
<script>mkdir -p /usr/sidewinder/bin</script>
289318
<script>mkdir -p /etc/sidewinder</script>
319+
<script>sysctl -w vm.max_map_count=262144</script>
290320
</preinstallScriptlet>
291321
<postinstallScriptlet>
292322
<script>chmod +x /usr/sidewinder/bin/sidewinder</script>

‎core/src/main/java/com/srotya/sidewinder/core/ConfigConstants.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,6 @@ public interface ConfigConstants {
3232
public static final String GRPC_EXECUTOR_COUNT = "grpc.executor.count";
3333
public static final String ENABLE_GRPC = "grpc.enabled";
3434
public static final String BG_THREAD_COUNT = "bgthread.count";
35-
public static final String GRAPHITE_ENABLED = "graphite.enabled";
35+
public static final String EXTERNAL_FUNCTIONS = "function.package.names";
3636

3737
}

‎core/src/main/java/com/srotya/sidewinder/core/SidewinderDropwizardReporter.java‎

Lines changed: 14 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919
import java.util.Arrays;
2020
import java.util.Map.Entry;
2121
import java.util.SortedMap;
22+
import java.util.concurrent.ScheduledExecutorService;
2223
import java.util.concurrent.TimeUnit;
2324

2425
import com.codahale.metrics.Counter;
@@ -43,8 +44,8 @@ public class SidewinderDropwizardReporter extends ScheduledReporter {
4344
private String name;
4445

4546
public SidewinderDropwizardReporter(MetricRegistry registry, String name, MetricFilter filter, TimeUnit rateUnit,
46-
TimeUnit durationUnit, StorageEngine engine) {
47-
super(registry, name, filter, rateUnit, durationUnit);
47+
TimeUnit durationUnit, StorageEngine engine, ScheduledExecutorService es) {
48+
super(registry, name, filter, rateUnit, durationUnit, es);
4849
this.name = name;
4950
this.engine = engine;
5051
}
@@ -74,6 +75,17 @@ public void report(@SuppressWarnings("rawtypes") SortedMap<String, Gauge> gauges
7475
}
7576
}
7677
}
78+
if (timers != null) {
79+
for (Entry<String, Timer> entry : timers.entrySet()) {
80+
DataPoint dp = MiscUtils.buildDataPoint(_INTERNAL, name, entry.getKey(), Arrays.asList("local"),
81+
System.currentTimeMillis(), entry.getValue().getSnapshot().getMean());
82+
try {
83+
engine.writeDataPoint(dp);
84+
} catch (IOException e) {
85+
e.printStackTrace();
86+
}
87+
}
88+
}
7789
}
7890

7991
}

0 commit comments

Comments
 (0)