Skip to content

Commit 9921527

Browse files
LukasMolzbergerLukasMolzberger
committed
FSSuspensionCallbackImpl
1 parent 67acb45 commit 9921527

File tree

6 files changed

+241
-11
lines changed

6 files changed

+241
-11
lines changed

‎pom.xml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55
<groupId>network.aika</groupId>
66
<artifactId>aika</artifactId>
77
<packaging>jar</packaging>
8-
<version>1.4.15</version>
8+
<version>1.4.16</version>
99
<name>aika</name>
1010
<url>https://aika.network</url>
1111
<description>An Artificial Intelligence for Knowledge Acquisition</description>
@@ -30,7 +30,7 @@
3030
<url>https://github.com/aika-algorithm/aika</url>
3131
<connection>scm:git:git://github.com/aika-algorithm/aika.git</connection>
3232
<developerConnection>scm:git:[email protected]:aika-algorithm/aika.git</developerConnection>
33-
<tag>aika-1.4.15</tag>
33+
<tag>aika-1.4.16</tag>
3434
</scm>
3535

3636

‎src/main/java/network/aika/Provider.java‎

Lines changed: 13 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -147,7 +147,11 @@ public void save() {
147147
}
148148
}
149149

150-
model.suspensionHook.store(id, n.getLabel(), n.getModelLabels(), n.isNeuron(), baos.toByteArray());
150+
try {
151+
model.suspensionHook.store(id, baos.toByteArray());
152+
} catch (IOException e) {
153+
throw new RuntimeException(e);
154+
}
151155
}
152156
n.modified = false;
153157
}
@@ -156,7 +160,12 @@ public void save() {
156160
private void reactivate() {
157161
assert model.suspensionHook != null;
158162

159-
byte[] data = model.suspensionHook.retrieve(id);
163+
byte[] data = new byte[0];
164+
try {
165+
data = model.suspensionHook.retrieve(id);
166+
} catch (IOException e) {
167+
throw new RuntimeException(e);
168+
}
160169
if(data == null) {
161170
log.warn("Tried to reactivate deleted node!");
162171
markedDeleted = true;
@@ -197,7 +206,8 @@ public void delete(Set<String> modelLabels) {
197206
n.delete(modelLabels);
198207

199208
// model.removeProvider(this);
200-
model.suspensionHook.delete(n.getLabel(), id);
209+
model.suspensionHook.remove(id);
210+
model.suspensionHook.removeLabel(n.getLabel());
201211
markedDeleted = true;
202212
}
203213

‎src/main/java/network/aika/SuspensionHook.java‎

Lines changed: 13 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717
package network.aika;
1818

1919

20+
import java.io.IOException;
2021
import java.util.Set;
2122

2223
/**
@@ -30,14 +31,23 @@
3031
*/
3132
public interface SuspensionHook {
3233

34+
Integer getIdByLabel(String label);
35+
36+
void putLabel(String label, Integer id);
37+
38+
void removeLabel(String label);
3339

3440
int getNewId();
3541

36-
void store(int id, String label, Set<String> modelLabels, boolean isNeuron, byte[] data);
42+
void store(Integer id, byte[] data) throws IOException;
3743

38-
byte[] retrieve(int id);
44+
byte[] retrieve(Integer id) throws IOException;
3945

40-
void delete(String label, int id);
46+
void remove(Integer id);
4147

4248
Iterable<Integer> getAllNodeIds();
49+
50+
void loadIndex();
51+
52+
void storeIndex();
4353
}

‎src/main/java/network/aika/storage/FSSuspensionCallbackImpl.java‎

Lines changed: 175 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,175 @@
1+
package network.aika.storage;
2+
3+
/*
4+
* Licensed to the Apache Software Foundation (ASF) under one or more
5+
* contributor license agreements. See the NOTICE file distributed with
6+
* this work for additional information regarding copyright ownership.
7+
* The ASF licenses this file to You under the Apache License, Version 2.0
8+
* (the "License"); you may not use this file except in compliance with
9+
* the License. You may obtain a copy of the License at
10+
*
11+
* http://www.apache.org/licenses/LICENSE-2.0
12+
*
13+
* Unless required by applicable law or agreed to in writing, software
14+
* distributed under the License is distributed on an "AS IS" BASIS,
15+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16+
* See the License for the specific language governing permissions and
17+
* limitations under the License.
18+
*/
19+
20+
import network.aika.SuspensionHook;
21+
22+
import java.io.*;
23+
import java.util.Collections;
24+
import java.util.Map;
25+
import java.util.TreeMap;
26+
import java.util.concurrent.atomic.AtomicInteger;
27+
28+
/**
29+
*
30+
* @author Lukas Molzberger
31+
*/
32+
public class FSSuspensionCallbackImpl implements SuspensionHook {
33+
34+
private AtomicInteger currentId = new AtomicInteger(0);
35+
36+
private Map<String, Integer> labels = Collections.synchronizedMap(new TreeMap<>());
37+
private Map<Integer, long[]> index = Collections.synchronizedMap(new TreeMap<>());
38+
39+
private File path;
40+
private String modelLabel;
41+
42+
private RandomAccessFile dataStore;
43+
44+
45+
public FSSuspensionCallbackImpl(File path, String modelLabel) throws FileNotFoundException {
46+
this.path = path;
47+
this.modelLabel = modelLabel;
48+
loadIndex();
49+
50+
dataStore = new RandomAccessFile(getFile("model"), "rw");
51+
}
52+
53+
@Override
54+
public Integer getIdByLabel(String label) {
55+
return labels.get(label);
56+
}
57+
58+
@Override
59+
public void putLabel(String label, Integer id) {
60+
labels.put(label, id);
61+
}
62+
63+
@Override
64+
public void removeLabel(String label) {
65+
if (label == null)
66+
return;
67+
68+
labels.remove(label);
69+
}
70+
71+
@Override
72+
public int getNewId() {
73+
return currentId.addAndGet(1);
74+
}
75+
76+
@Override
77+
public synchronized void store(Integer id, byte[] data) throws IOException {
78+
dataStore.seek(dataStore.length());
79+
80+
index.put(id, new long[]{(int) dataStore.getFilePointer(), data.length});
81+
dataStore.write(data);
82+
}
83+
84+
@Override
85+
public synchronized byte[] retrieve(Integer id) throws IOException {
86+
long[] pos = index.get(id);
87+
if(pos == null)
88+
throw new MissingNodeException(String.format("Neuron with id %d is missing in model label %d", id, modelLabel));
89+
90+
byte[] data = new byte[(int)pos[1]];
91+
92+
dataStore.seek(pos[0]);
93+
dataStore.read(data);
94+
95+
return data;
96+
}
97+
98+
@Override
99+
public synchronized void remove(Integer id) {
100+
index.remove(id);
101+
}
102+
103+
@Override
104+
public Iterable<Integer> getAllNodeIds() {
105+
return index.keySet();
106+
}
107+
108+
@Override
109+
public void loadIndex() {
110+
try (FileInputStream fis = new FileInputStream(getFile("index"));
111+
ByteArrayInputStream bais = new ByteArrayInputStream(fis.readAllBytes());
112+
DataInputStream dis = new DataInputStream(bais)) {
113+
readIndex(dis);
114+
} catch (Exception e) {
115+
throw new RuntimeException(e);
116+
}
117+
}
118+
119+
@Override
120+
public void storeIndex() {
121+
ByteArrayOutputStream baos = new ByteArrayOutputStream();
122+
123+
try (DataOutputStream dos = new DataOutputStream(baos);
124+
FileOutputStream fos = new FileOutputStream(getFile("index"))) {
125+
writeIndex(dos);
126+
fos.write(baos.toByteArray());
127+
} catch (IOException e) {
128+
throw new RuntimeException(e);
129+
}
130+
}
131+
132+
private File getFile(String prefix) {
133+
return new File(path, prefix + "-" + modelLabel + ".dat");
134+
}
135+
136+
private void readIndex(DataInput in) throws IOException {
137+
currentId = new AtomicInteger(in.readInt());
138+
139+
labels.clear();
140+
while(in.readBoolean()) {
141+
String l = in.readUTF();
142+
Integer id = in.readInt();
143+
labels.put(l, id);
144+
}
145+
146+
index.clear();
147+
while(in.readBoolean()) {
148+
Integer id = in.readInt();
149+
long[] pos = new long[2];
150+
pos[0] = in.readLong();
151+
pos[1] = in.readInt();
152+
153+
index.put(id, pos);
154+
}
155+
}
156+
157+
private void writeIndex(DataOutput out) throws IOException {
158+
out.writeLong(currentId.get());
159+
160+
for(Map.Entry<String, Integer> me: labels.entrySet()) {
161+
out.writeBoolean(true);
162+
out.writeUTF(me.getKey());
163+
out.writeLong(me.getValue());
164+
}
165+
out.writeBoolean(false);
166+
167+
for(Map.Entry<Integer, long[]> me: index.entrySet()) {
168+
out.writeBoolean(true);
169+
out.writeLong(me.getKey());
170+
out.writeLong(me.getValue()[0]);
171+
out.writeInt((int)me.getValue()[1]);
172+
}
173+
out.writeBoolean(false);
174+
}
175+
}

‎src/main/java/network/aika/storage/MissingNodeException.java‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
package network.aika.storage;
2+
3+
public class MissingNodeException extends RuntimeException {
4+
5+
public MissingNodeException(String message) {
6+
super(message);
7+
}
8+
}

‎src/test/java/network/aika/network/SuspensionTest.java‎

Lines changed: 30 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -133,31 +133,58 @@ public static class DummySuspensionHook implements SuspensionHook {
133133
public AtomicInteger currentId = new AtomicInteger(0);
134134

135135
Map<Integer, byte[]> storage = new TreeMap<>();
136+
private Map<String, Integer> labels = new TreeMap<>();
137+
136138

137139
@Override
138140
public int getNewId() {
139141
return currentId.addAndGet(1);
140142
}
141143

142144
@Override
143-
public void store(int id, String label, Set<String> modelLabels, boolean isNeuron, byte[] data) {
145+
public void store(Integer id, byte[] data) {
144146
storage.put(id, data);
145147
}
146148

147149
@Override
148-
public byte[] retrieve(int id) {
150+
public byte[] retrieve(Integer id) {
149151
return storage.get(id);
150152
}
151153

152154
@Override
153-
public void delete(String label, int id) {
155+
public void remove(Integer id) {
154156
storage.remove(id);
155157
}
156158

159+
@Override
160+
public Integer getIdByLabel(String label) {
161+
return labels.get(label);
162+
}
163+
164+
@Override
165+
public void putLabel(String label, Integer id) {
166+
labels.put(label, id);
167+
}
168+
169+
@Override
170+
public void removeLabel(String label) {
171+
labels.remove(label);
172+
}
173+
157174
@Override
158175
public Iterable<Integer> getAllNodeIds() {
159176
return storage.keySet();
160177
}
178+
179+
@Override
180+
public void loadIndex() {
181+
182+
}
183+
184+
@Override
185+
public void storeIndex() {
186+
187+
}
161188
}
162189

163190

0 commit comments

Comments
 (0)