You signed in with another tab or window. Reload to refresh your session.You signed out in another tab or window. Reload to refresh your session.You switched accounts on another tab or window. Reload to refresh your session.Dismiss alert
Tool designed for consistent data storage in B2 and database
Simple initialization examples:
Writer:
Used to write data to a database by creating a blob with an id (fid)
in which N messages are also stored with their own unique id
funcmain() {
db, err:=gorm.Open(mysql.Open(""), &gorm.Config{})
iferr!=nil {
panic(err)
}
w, err:=axq.NewWriter().
WithName("writer_name").
WithDB(db).
// MaxBlobSize specify max messages in blob (default 10000)WithMaxBlobSize(10000).
// ChunkSize specify how much messages in chunk (default 1000)WithChunkSize(1000).
// WithoutCompression specify if you not want to compress yourWithoutCompression().
Build()
iferr!=nil {
panic(err)
}
err=w.Push([]byte("some msg"))
iferr!=nil {
panic(err)
}
}
Reader:
Used to read data from Database and B2 if credentials
provided. Reading batches of blobs from db then sorting
before send to outer. Ordered number of messages guaranteed.
Note that message id that can be sent to outer should be > reader.lastId
db, err:=gorm.Open(mysql.Open(""), &gorm.Config{})
iferr!=nil {
panic(err)
}
r, err:=axq.NewReader().
WithName("reader_name").
WithDB(db).
// LoaderCount start N loader workersWithLoaderCount(2).
// WaiterCount start N outers processing sorter messageWithWaiterCount(2).
// BufferSize specify size of buffer chan for message queueWithBufferSize(100_000).
// Provide only if you need read b2 before start reading dbWithB2Credentials(creds).
Build()
iferr!=nil {
panic(err)
}
for {
m:=r.Pop()
// Handle messagem.Done()
}
Archiver
Used to archive data from database and send it as compressed and
encrypted blob to B2
funcmain() {
db, err:=gorm.Open(mysql.Open(""), &gorm.Config{})
iferr!=nil {
panic(err)
}
// B2 Application Key and Key ID should be exported to envcreds:= backblaze.Credentials{
ApplicationKey: os.Getenv("B2_APP_KEY"),
KeyID: os.Getenv("B2_KEY_ID"),
}
a, err:=axq.NewArchiver().
WithName("archiver_name").
WithDB(db).
WithB2Credentials(creds).
// MaxCount specify max messages in blobWithMaxCount(5_000_000). // Default 5_000_000// MaxCount specify max size of blobWithMaxSize(1_000_000). // Default 1_000_000// ChunkSize specify how often blob size recalculation will be madeWithChunkSize(2_000). // Default 2_000Build()
defera.Close()
}
B2 Reader
db, err:=gorm.Open(mysql.Open(""), &gorm.Config{})
iferr!=nil {
panic(err)
}
// B2 Application Key and Key ID should be exported to envcreds:= backblaze.Credentials{
ApplicationKey: os.Getenv("B2_APP_KEY"),
KeyID: os.Getenv("B2_KEY_ID"),
}
r, err:=axq.NewB2Reader().
WithName("reader_name").
// LoaderCount start N loader workersWithLoaderCount(2).
// WaiterCount start N outers processing sorter messageWithOuterCount(2).
// BufferSize specify size of buffer chan for message queueWithBufferSize(100_000).
// Provide only if you need read b2 before start reading dbWithB2Credentials(creds).
Build()
iferr!=nil {
panic(err)
}
for {
m:=r.Pop()
// Handle messagem.Done()
}