Skip to content

Commit 9115d08

Browse files
gencebaygencebay
committed
transform configs, multiple tasks, dockerfile, updates
1 parent e920a5c commit 9115d08

File tree

19 files changed

+220
-150
lines changed

19 files changed

+220
-150
lines changed

‎DataTransform.sln‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,9 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "DataTransform.SharedLibrary
1515
EndProject
1616
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution Items", "{74675BBC-305B-49AC-B472-19A4AD99A801}"
1717
ProjectSection(SolutionItems) = preProject
18+
afterdockerbuild.ps1 = afterdockerbuild.ps1
19+
afterdockerbuild.sh = afterdockerbuild.sh
20+
Dockerfile = Dockerfile
1821
README.md = README.md
1922
EndProjectSection
2023
EndProject

‎Dockerfile‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,10 @@ RUN mkdir -p /var/opt/mssql/backup
44

55
COPY MusicStore.bak /var/opt/mssql/backup
66

7+
##Enable it if you have already downloaded the .bak file for this database
8+
##https://github.com/Microsoft/sql-server-samples/releases/download/adventureworks/AdventureWorks2016.bak
9+
#COPY AdventureWorks2016.bak /var/opt/mssql/backup
10+
711
ENV MSSQL_SA_PASSWORD=P@ssword1
812

913
ENV ACCEPT_EULA=Y

‎README.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@
66

77
docker run -it -v mongodata:/data/db -p 27017:27017 -d mongo
88

9-
### MSSQL Linux Database
9+
### MSSQL Linux Database (Default MusicStore)
1010

1111
docker build -t localsql .
1212

‎src/DataTransform.Api.Hosting/Controllers/TransformController.cs‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -131,12 +131,17 @@ public IActionResult CreateTransformFile([FromBody] SaveFileModel model)
131131
}
132132

133133
[HttpGet(nameof(StartTransformAsync))]
134-
public async Task<IActionResult> StartTransformAsync([FromQuery] string filename)
134+
public async Task<IActionResult> StartTransformAsync([FromQuery] string[] files)
135135
{
136+
if (files == null || !files.Any())
137+
{
138+
return BadRequest();
139+
}
140+
136141
var transformManager = HttpContext.RequestServices.GetRequiredService<TransformManager>();
137142

138143
#pragma warning disable CS4014 // Because this call is not awaited, execution of the current method continues before the call is completed
139-
Task.Factory.StartNew(async () => await transformManager.TransformAsync(filename), TaskCreationOptions.LongRunning);
144+
Task.Factory.StartNew(async () => await transformManager.TransformAsync(files), TaskCreationOptions.LongRunning);
140145
#pragma warning restore CS4014 // Because this call is not awaited, execution of the current method continues before the call is completed
141146

142147
await Task.CompletedTask;

‎src/DataTransform.Api.Hosting/Core/DbTransformTask.cs‎

Lines changed: 27 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,15 @@
11
using Dapper;
22
using MongoDB.Bson;
33
using MongoDB.Driver;
4+
using NetCoreStack.Data;
5+
using NetCoreStack.Data.Context;
46
using NetCoreStack.Data.Interfaces;
57
using NetCoreStack.WebSockets;
68
using System;
79
using System.Collections.Generic;
810
using System.Diagnostics;
911
using System.Linq;
12+
using System.Threading;
1013
using System.Threading.Tasks;
1114

1215
namespace DataTransform.Api.Hosting
@@ -15,15 +18,28 @@ public class DbTransformTask : ITransformTask
1518
{
1619
private readonly SqlDatabase _sourceSqlDatabase;
1720
private readonly IMongoDbDataContext _mongoDbDataContext;
21+
private readonly TransformOptions _options;
22+
private readonly ICollectionNameSelector _collectionNameSelector;
1823
private readonly IConnectionManager _connectionManager;
24+
private readonly CancellationTokenSource _cancellationToken;
1925

20-
public DbTransformTask(SqlDatabase sourceSqlDatabase,
21-
IMongoDbDataContext mongoDbDataContext,
22-
IConnectionManager connectionManager)
26+
public List<DbTransformContext> DbTransformContexts { get; }
27+
28+
public DbTransformTask(TransformOptions options,
29+
ICollectionNameSelector collectionNameSelector,
30+
IConnectionManager connectionManager,
31+
CancellationTokenSource cancellationToken)
2332
{
24-
_sourceSqlDatabase = sourceSqlDatabase;
25-
_mongoDbDataContext = mongoDbDataContext;
26-
_connectionManager = connectionManager;
33+
_options = options ?? throw new ArgumentNullException(nameof(options));
34+
_collectionNameSelector = collectionNameSelector ?? throw new ArgumentNullException(nameof(collectionNameSelector));
35+
_connectionManager = connectionManager ?? throw new ArgumentNullException(nameof(connectionManager));
36+
_cancellationToken = cancellationToken ?? throw new ArgumentNullException(nameof(cancellationToken));
37+
38+
var dataContextConfigurationAccessor = new DefaultDataContextConfigurationAccessor(options);
39+
_sourceSqlDatabase = new SqlDatabase(dataContextConfigurationAccessor);
40+
_mongoDbDataContext = new MongoDbContext(dataContextConfigurationAccessor, _collectionNameSelector, null);
41+
42+
DbTransformContexts = options.CreateTransformContexts(_cancellationToken);
2743
}
2844

2945
private long GetCount(DbTransformContext context)
@@ -116,9 +132,12 @@ private async Task InvokeInternal(DbTransformContext context)
116132
await _connectionManager.WsLogAsync(string.Format("Transformed total records: {0} time elapsed: {1}", totalRecords, sw.Elapsed));
117133
}
118134

119-
public async Task InvokeAsync(DbTransformContext context)
135+
public async Task InvokeAsync()
120136
{
121-
await InvokeInternal(context);
137+
foreach (var context in DbTransformContexts)
138+
{
139+
await InvokeInternal(context);
140+
}
122141
}
123142
}
124143
}

‎src/DataTransform.Api.Hosting/Core/TransformManager.cs‎

Lines changed: 9 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,6 @@
1-
using Microsoft.AspNetCore.Hosting;
2-
using Microsoft.Extensions.Configuration;
3-
using NetCoreStack.Data;
4-
using NetCoreStack.Data.Context;
5-
using NetCoreStack.WebSockets;
1+
using NetCoreStack.WebSockets;
62
using System;
7-
using System.Collections.Generic;
8-
using System.IO;
3+
using System.Linq;
94
using System.Threading;
105
using System.Threading.Tasks;
116
using static DataTransform.SharedLibrary.HostingConstants;
@@ -14,49 +9,26 @@ namespace DataTransform.Api.Hosting
149
{
1510
public class TransformManager
1611
{
17-
private readonly IHostingEnvironment _hostingEnvironment;
1812
private readonly IConnectionManager _connectionManager;
19-
private readonly ICollectionNameSelector _collectionNameSelector;
13+
private readonly TransformTaskFactory _transformTaskFactory;
2014

2115
public CancellationTokenSource CancellationToken { get; }
2216

23-
public TransformManager(IHostingEnvironment hostingEnvironment,
24-
IConnectionManager connectionManager,
25-
ICollectionNameSelector collectionNameSelector)
17+
public TransformManager(TransformTaskFactory transformTaskFactory, IConnectionManager connectionManager)
2618
{
27-
_hostingEnvironment = hostingEnvironment ?? throw new ArgumentNullException(nameof(hostingEnvironment));
19+
_transformTaskFactory = transformTaskFactory ?? throw new ArgumentNullException(nameof(transformTaskFactory));
2820
_connectionManager = connectionManager ?? throw new ArgumentNullException(nameof(connectionManager));
29-
_collectionNameSelector = collectionNameSelector;
3021
CancellationToken = new CancellationTokenSource();
3122
}
3223

33-
public async Task TransformAsync(string filename)
34-
{
35-
var configFilePath = Path.Combine(_hostingEnvironment.WebRootPath, "configs", filename);
36-
if (!System.IO.File.Exists(configFilePath))
37-
{
38-
throw new FileNotFoundException($"{configFilePath} not found.");
39-
}
24+
public async Task TransformAsync(string[] files)
25+
{
4026

4127
SharedSemaphoreSlim.Wait();
4228
try
4329
{
44-
var configuration = new ConfigurationBuilder().AddJsonFile(configFilePath).Build();
45-
var options = new TransformOptions();
46-
configuration.Bind(nameof(TransformOptions), options);
47-
48-
List<DbTransformContext> transformContexts = options.CreateTransformContexts(CancellationToken);
49-
50-
var dataContextConfigurationAccessor = new DefaultDataContextConfigurationAccessor(options);
51-
var sqlDatabase = new SqlDatabase(dataContextConfigurationAccessor);
52-
var mongoDbContext = new MongoDbContext(dataContextConfigurationAccessor, _collectionNameSelector, null);
53-
54-
ITransformTask transformTask = new DbTransformTask(sqlDatabase, mongoDbContext, _connectionManager);
55-
56-
foreach (var context in transformContexts)
57-
{
58-
await transformTask.InvokeAsync(context);
59-
}
30+
var tasks = _transformTaskFactory.Create(files, CancellationToken);
31+
await Task.WhenAll(tasks.Select(t => t.InvokeAsync()));
6032
}
6133
catch (Exception ex)
6234
{

‎src/DataTransform.Api.Hosting/Core/TransformTaskFactory.cs‎

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
using Microsoft.AspNetCore.Hosting;
2+
using Microsoft.Extensions.Configuration;
3+
using NetCoreStack.Data;
4+
using NetCoreStack.WebSockets;
5+
using System.Collections.Generic;
6+
using System.IO;
7+
using System.Threading;
8+
9+
namespace DataTransform.Api.Hosting
10+
{
11+
public class TransformTaskFactory
12+
{
13+
private readonly IHostingEnvironment _hostingEnvironment;
14+
private readonly IConnectionManager _connectionManager;
15+
private readonly ICollectionNameSelector _collectionNameSelector;
16+
17+
public TransformTaskFactory(IHostingEnvironment hostingEnvironment,
18+
IConnectionManager connectionManager,
19+
ICollectionNameSelector collectionNameSelector)
20+
{
21+
_hostingEnvironment = hostingEnvironment;
22+
_connectionManager = connectionManager;
23+
_collectionNameSelector = collectionNameSelector;
24+
}
25+
26+
public List<ITransformTask> Create(string[] files, CancellationTokenSource cancellationToken)
27+
{
28+
List<ITransformTask> taskList = new List<ITransformTask>();
29+
foreach (var file in files)
30+
{
31+
var configFilePath = Path.Combine(_hostingEnvironment.WebRootPath, "configs", file);
32+
if (!File.Exists(configFilePath))
33+
{
34+
throw new FileNotFoundException($"{configFilePath} not found.");
35+
}
36+
37+
var configuration = new ConfigurationBuilder().AddJsonFile(configFilePath).Build();
38+
var options = new TransformOptions();
39+
configuration.Bind(nameof(TransformOptions), options);
40+
41+
taskList.Add(new DbTransformTask(options, _collectionNameSelector, _connectionManager, cancellationToken));
42+
}
43+
44+
return taskList;
45+
}
46+
}
47+
}

‎src/DataTransform.Api.Hosting/Extensions/ServiceCollectionExtensions.cs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,8 @@ public static void AddTransformFeatures(this IServiceCollection services)
1212

1313
services.TryAddSingleton<ICollectionNameSelector, DefaultCollectionNameSelector>();
1414

15+
services.AddSingleton<TransformTaskFactory>();
16+
1517
services.AddSingleton<TransformManager>();
1618
}
1719
}

‎src/DataTransform.Api.Hosting/Extensions/TransformOptionsExtensions.cs‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,6 @@
1-
using System;
1+
using Microsoft.Extensions.Configuration;
22
using System.Collections.Generic;
3-
using System.Linq;
43
using System.Threading;
5-
using System.Threading.Tasks;
64

75
namespace DataTransform.Api.Hosting
86
{

‎src/DataTransform.Api.Hosting/Helpers/PathUtility.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ public static List<JsTreeDataModel> WalkDirectoryTree(DirectoryInfo directory, J
2020
{
2121
Text = fi.Name,
2222
Id = fi.Name,
23-
Icon = "file file" + extension,
23+
Icon = "file file-js",
2424
Type = "file"
2525
});
2626
}

0 commit comments

Comments
 (0)