Skip to content

Commit ff03f74

Browse files
kordanokordano
authored
Merge pull request #9 from lambdaforge/datomic-cloud-tx
use the tx log for Datomic Cloud extraction
2 parents 9bb49a6 + 25a18f9 commit ff03f74

File tree

4 files changed

+54
-57
lines changed

4 files changed

+54
-57
lines changed

‎project.clj‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -5,15 +5,15 @@
55
:url "https://www.eclipse.org/legal/epl-2.0/"}
66
:dependencies [[org.clojure/clojure "1.10.1"]
77
[io.replikativ/datahike-jdbc "0.1.2-SNAPSHOT"]
8-
[io.replikativ/datahike "0.3.3-SNAPSHOT" :exclusions [io.replikativ/konserve]]
9-
[com.datomic/client-cloud "0.8.78" :scope "provided"]
8+
[io.replikativ/datahike "0.3.7-SNAPSHOT" :exclusions [io.replikativ/konserve]]
9+
[com.datomic/client-cloud "0.8.105" :scope "provided"]
1010
[com.datomic/client-pro "0.9.63" :scope "provided"]
11-
[com.datomic/dev-local "0.9.232"]
1211
[com.cognitect/transit-clj "0.8.313"]
1312
[com.taoensso/nippy "3.1.1"]
1413
[org.clojure/tools.cli "1.0.206"]]
1514
:resource-paths ["resources"]
1615
:profiles {:dev {:dependencies [[org.clojure/test.check "0.9.0"]]}
17-
:test {:dependencies [[lambdaisland/kaocha "1.0.632"]]}}
16+
:test {:dependencies [[com.datomic/dev-local "0.9.232"]
17+
[lambdaisland/kaocha "1.0.632"]]}}
1818
:main wanderung.core
1919
:repl-options {:init-ns wanderung.core})

‎src/wanderung/core.clj‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -20,8 +20,11 @@
2020

2121
;;; Datomic
2222
(defn datomic-connect [datomic-config]
23-
(dt/connect (dt/client (dissoc datomic-config :name))
24-
{:db-name (:name datomic-config)}))
23+
(println "➜ Connecting to Datomic...")
24+
(let [result (dt/connect (dt/client (dissoc datomic-config :name))
25+
{:db-name (:name datomic-config)})]
26+
(println " ✓ Done")
27+
result))
2528

2629
(defmethod datoms-from-storage :datomic [storage]
2730
(-> storage
@@ -58,7 +61,8 @@
5861
wd/extract-datahike-data))
5962

6063
(defmethod datoms-to-storage :datahike [storage datoms]
61-
@(d/load-entities (datahike-maybe-create-and-connect storage) datoms))
64+
@(d/load-entities (datahike-maybe-create-and-connect storage) datoms)
65+
true)
6266

6367
;;;------- Migrations -------
6468

‎src/wanderung/datomic_cloud.clj‎

Lines changed: 25 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -92,35 +92,29 @@
9292
:tempids {}}
9393
transaction-groups)))
9494

95-
96-
97-
(defn extract-datomic-cloud-data [conn]
95+
(defn create-schema-mapping [conn]
9896
(let [db (d/db conn)
99-
txs (sort-by first (d/q '[:find ?tx ?inst
100-
:in $ ?bf
101-
:where
102-
[?tx :db/txInstant ?inst]
103-
[(< ?bf ?inst)]]
104-
db
105-
(java.util.Date. 70)))
106-
schema-attrs #{:db/cardinality :db/valueType :db/unique}
107-
id->ident (->> (d/q '[:find ?e ?id :where [?e :db/ident ?id]] db)
108-
(into {}))
109-
query {:query '[:find ?e ?at ?v ?t ?added
110-
:in $ ?t
111-
:where
112-
[?e ?a ?v ?t ?added]
113-
[?a :db/ident ?at]
114-
(not [?a :db/ident :db/txInstant])]
115-
:args [(d/history db)]}]
116-
(letfn [(update-schema-attr [[_ a _ _ _ :as entity]]
117-
(if (schema-attrs a)
118-
(update entity 2 id->ident)
119-
entity))]
120-
(mapcat
121-
(fn [[tid tinst]]
122-
(->> (d/q (update-in query [:args] conj tid))
123-
(map update-schema-attr)
124-
(sort-by first)
125-
(into [[tid :db/txInstant tinst tid true]])))
126-
txs))))
97+
query '[:find ?e ?ident
98+
:where
99+
[?e :db/ident ?ident]]]
100+
(into {} (d/q query db))))
101+
102+
(defn extract-datomic-cloud-data
103+
"Extracts all transactions from Datomic with keyword attributes given a Datomic connection.
104+
Internally Datomic uses the first transactions to initialize the system schema and identifiers
105+
which are Datomic specific and not relevant for import.
106+
Currently, it takes 5 transactions, so the 6th is the first user specific one."
107+
[conn]
108+
(let [system-attributes #{:db.install/valueType :db/valueType :db/cardinality :db/unique}
109+
start-tx 6 ;; first user transaction
110+
schema-mapping (create-schema-mapping conn)
111+
map-db-ident (map (fn [[e a v tx added]]
112+
(let [new-a (schema-mapping a)]
113+
(assert new-a)
114+
[e new-a (if (system-attributes new-a)
115+
(schema-mapping v)
116+
v) tx added])))
117+
data-extract (mapcat (fn [{:keys [data]}]
118+
(into [] map-db-ident data)))
119+
tx-data (d/tx-range conn {:start start-tx})]
120+
(into [] data-extract tx-data)))

‎test/wanderung/core_test.clj‎

Lines changed: 18 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -5,8 +5,7 @@
55
[datomic.client.api :as dt]
66
[clojure.edn :as edn]
77
[taoensso.timbre :as timbre]
8-
[clojure.java.io :as io]
9-
[datahike.api :as d])
8+
[clojure.java.io :as io])
109
(:import [java.io File]))
1110

1211
(timbre/set-level! :warn)
@@ -48,24 +47,24 @@
4847

4948
(defn setup-data [tx-fn conn]
5049
(tx-fn conn {:tx-data [{:db/ident :person/name
51-
:db/valueType :db.type/string
52-
:db/unique :db.unique/identity
53-
:db/cardinality :db.cardinality/one}
54-
{:db/ident :person/age
55-
:db/valueType :db.type/long
56-
:db/cardinality :db.cardinality/one}
57-
{:db/ident :person/siblings
58-
:db/valueType :db.type/ref
59-
:db/cardinality :db.cardinality/many}]})
50+
:db/valueType :db.type/string
51+
:db/unique :db.unique/identity
52+
:db/cardinality :db.cardinality/one}
53+
{:db/ident :person/age
54+
:db/valueType :db.type/long
55+
:db/cardinality :db.cardinality/one}
56+
{:db/ident :person/siblings
57+
:db/valueType :db.type/ref
58+
:db/cardinality :db.cardinality/many}]})
6059
(tx-fn conn {:tx-data [{:db/id -1
61-
:person/name "Alice"
62-
:person/age 25}
63-
{:db/id -2
64-
:person/name "Bob"
65-
:person/age 35}
66-
{:person/name "Charlie"
67-
:person/age 45
68-
:person/siblings [-1 -2]}]}))
60+
:person/name "Alice"
61+
:person/age 25}
62+
{:db/id -2
63+
:person/name "Bob"
64+
:person/age 35}
65+
{:person/name "Charlie"
66+
:person/age 45
67+
:person/siblings [-1 -2]}]}))
6968

7069
(deftest test-nippy-migration
7170
(let [a {:wanderung/type :nippy

0 commit comments

Comments
 (0)