Skip to content

Commit 8cccb72

Browse files
egorpuginegorpugin
committed
Switch to strong file hash.
1 parent c876a31 commit 8cccb72

10 files changed

Lines changed: 81 additions & 43 deletions

File tree

‎src/common/database.cpp‎

Lines changed: 21 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -63,7 +63,7 @@ std::vector<StartupAction> startup_actions{
6363
{ 7, StartupAction::ClearStorageDirExp | StartupAction::ClearStorageDirBin | StartupAction::ClearStorageDirLib },
6464
{ 8, StartupAction::ClearCfgDirs },
6565
{ 9, StartupAction::ClearStorageDirExp },
66-
// 10
66+
{ 10, StartupAction::ClearPackagesDatabase },
6767
};
6868

6969
const TableDescriptors &get_service_tables()
@@ -211,7 +211,7 @@ const TableDescriptors data_tables{
211211
"branch" TEXT,
212212
"flags" INTEGER NOT NULL,
213213
"created" DATE NOT NULL,
214-
"sha256" TEXT NOT NULL,
214+
"hash" TEXT NOT NULL,
215215
PRIMARY KEY ("id"),
216216
FOREIGN KEY ("project_id") REFERENCES "Projects" ("id")
217217
);
@@ -477,6 +477,11 @@ void ServiceDatabase::performStartupActions() const
477477
}
478478
}
479479

480+
if (a.action & StartupAction::ClearPackagesDatabase)
481+
{
482+
fs::remove(getDbDirectory() / packages_db_name);
483+
}
484+
480485
if (a.action & StartupAction::ClearStorageDirExp)
481486
{
482487
remove_all_from_dir(directories.storage_dir_exp);
@@ -1076,7 +1081,7 @@ IdDependencies PackagesDatabase::findDependencies(const Packages &deps) const
10761081
auto find_deps = [&dep, &all_deps, this](auto &dependency)
10771082
{
10781083
dependency.flags.set(pfDirectDependency);
1079-
dependency.id = getExactProjectVersionId(dependency, dependency.version, dependency.flags, dependency.sha256);
1084+
dependency.id = getExactProjectVersionId(dependency, dependency.version, dependency.flags, dependency.hash);
10801085
all_deps[dependency] = dependency; // assign first, deps assign second
10811086
all_deps[dependency].db_dependencies = getProjectDependencies(dependency.id, all_deps);
10821087
};
@@ -1136,7 +1141,7 @@ void check_version_age(const TimePoint &t1, const char *created)
11361141
throw std::runtime_error("One of the queried packages is 'young'. Young packages must be retrieved from server.");
11371142
}
11381143

1139-
ProjectVersionId PackagesDatabase::getExactProjectVersionId(const DownloadDependency &project, Version &version, ProjectFlags &flags, String &sha256) const
1144+
ProjectVersionId PackagesDatabase::getExactProjectVersionId(const DownloadDependency &project, Version &version, ProjectFlags &flags, String &hash) const
11401145
{
11411146
auto err = [](const auto &v, const auto &p)
11421147
{
@@ -1148,7 +1153,7 @@ ProjectVersionId PackagesDatabase::getExactProjectVersionId(const DownloadDepend
11481153
static auto tstart = getUtc();
11491154

11501155
ProjectVersionId id = 0;
1151-
static const String select = "select id, major, minor, patch, flags, sha256, created from ProjectVersions where ";
1156+
static const String select = "select id, major, minor, patch, flags, hash, created from ProjectVersions where ";
11521157

11531158
if (!version.isBranch())
11541159
{
@@ -1159,11 +1164,11 @@ ProjectVersionId PackagesDatabase::getExactProjectVersionId(const DownloadDepend
11591164
"project_id = '" + std::to_string(project.id) + "' and "
11601165
"major = '" + std::to_string(v.major) + "' and "
11611166
"minor = '" + std::to_string(v.minor) + "' and "
1162-
"patch = '" + std::to_string(v.patch) + "'", [&id, &flags, &sha256](SQLITE_CALLBACK_ARGS)
1167+
"patch = '" + std::to_string(v.patch) + "'", [&id, &flags, &hash](SQLITE_CALLBACK_ARGS)
11631168
{
11641169
id = std::stoull(cols[0]);
11651170
flags |= decltype(project.flags)(std::stoull(cols[4]));
1166-
sha256 = cols[5];
1171+
hash = cols[5];
11671172
check_version_age(tstart, cols[6]);
11681173
return 0;
11691174
});
@@ -1179,12 +1184,12 @@ ProjectVersionId PackagesDatabase::getExactProjectVersionId(const DownloadDepend
11791184
"major = '" + std::to_string(v.major) + "' and "
11801185
"minor = '" + std::to_string(v.minor) + "' and "
11811186
"branch is null order by major desc, minor desc, patch desc limit 1",
1182-
[&id, &version, &flags, &sha256](SQLITE_CALLBACK_ARGS)
1187+
[&id, &version, &flags, &hash](SQLITE_CALLBACK_ARGS)
11831188
{
11841189
id = std::stoull(cols[0]);
11851190
version.patch = std::stoi(cols[3]);
11861191
flags |= decltype(project.flags)(std::stoull(cols[4]));
1187-
sha256 = cols[5];
1192+
hash = cols[5];
11881193
check_version_age(tstart, cols[6]);
11891194
return 0;
11901195
});
@@ -1199,13 +1204,13 @@ ProjectVersionId PackagesDatabase::getExactProjectVersionId(const DownloadDepend
11991204
"project_id = '" + std::to_string(project.id) + "' and "
12001205
"major = '" + std::to_string(v.major) + "' and "
12011206
"branch is null order by major desc, minor desc, patch desc limit 1",
1202-
[&id, &version, &flags, &sha256](SQLITE_CALLBACK_ARGS)
1207+
[&id, &version, &flags, &hash](SQLITE_CALLBACK_ARGS)
12031208
{
12041209
id = std::stoull(cols[0]);
12051210
version.minor = std::stoi(cols[2]);
12061211
version.patch = std::stoi(cols[3]);
12071212
flags |= decltype(project.flags)(std::stoull(cols[4]));
1208-
sha256 = cols[5];
1213+
hash = cols[5];
12091214
check_version_age(tstart, cols[6]);
12101215
return 0;
12111216
});
@@ -1219,14 +1224,14 @@ ProjectVersionId PackagesDatabase::getExactProjectVersionId(const DownloadDepend
12191224
select + " "
12201225
"project_id = '" + std::to_string(project.id) + "' and "
12211226
"branch is null order by major desc, minor desc, patch desc limit 1",
1222-
[&id, &version, &flags, &sha256](SQLITE_CALLBACK_ARGS)
1227+
[&id, &version, &flags, &hash](SQLITE_CALLBACK_ARGS)
12231228
{
12241229
id = std::stoull(cols[0]);
12251230
version.major = std::stoi(cols[1]);
12261231
version.minor = std::stoi(cols[2]);
12271232
version.patch = std::stoi(cols[3]);
12281233
flags |= decltype(project.flags)(std::stoull(cols[4]));
1229-
sha256 = cols[5];
1234+
hash = cols[5];
12301235
check_version_age(tstart, cols[6]);
12311236
return 0;
12321237
});
@@ -1245,11 +1250,11 @@ ProjectVersionId PackagesDatabase::getExactProjectVersionId(const DownloadDepend
12451250
db->execute(
12461251
select + " "
12471252
"project_id = '" + std::to_string(project.id) + "' and "
1248-
"branch = '" + version.toString() + "'", [&id, &flags, &sha256](SQLITE_CALLBACK_ARGS)
1253+
"branch = '" + version.toString() + "'", [&id, &flags, &hash](SQLITE_CALLBACK_ARGS)
12491254
{
12501255
id = std::stoull(cols[0]);
12511256
flags |= decltype(project.flags)(std::stoull(cols[4]));
1252-
sha256 = cols[5];
1257+
hash = cols[5];
12531258
check_version_age(tstart, cols[6]);
12541259
return 0;
12551260
});
@@ -1288,7 +1293,7 @@ PackagesDatabase::Dependencies PackagesDatabase::getProjectDependencies(ProjectV
12881293

12891294
for (auto &dependency : deps)
12901295
{
1291-
dependency.id = getExactProjectVersionId(dependency, dependency.version, dependency.flags, dependency.sha256);
1296+
dependency.id = getExactProjectVersionId(dependency, dependency.version, dependency.flags, dependency.hash);
12921297
auto i = dm.find(dependency);
12931298
if (i == dm.end())
12941299
{

‎src/common/database.h‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,7 @@ struct StartupAction
5050
ClearStorageDirBin = 0x0010,
5151
ClearStorageDirLib = 0x0020,
5252
ClearCfgDirs = 0x0040,
53+
ClearPackagesDatabase = 0x0080,
5354
};
5455

5556
int id;
@@ -166,7 +167,7 @@ class PackagesDatabase : public Database
166167

167168
bool isCurrentDbOld() const;
168169

169-
ProjectVersionId getExactProjectVersionId(const DownloadDependency &project, Version &version, ProjectFlags &flags, String &sha256) const;
170+
ProjectVersionId getExactProjectVersionId(const DownloadDependency &project, Version &version, ProjectFlags &flags, String &hash) const;
170171
Dependencies getProjectDependencies(ProjectVersionId project_version_id, DependenciesMap &dm) const;
171172
};
172173

‎src/common/dependency.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@ struct DownloadDependency : public Package
3131

3232
// extended data
3333
ProjectVersionId id = 0;
34-
String sha256;
34+
String hash;
3535

3636
// own data (private)
3737
const Remote *remote = nullptr;

‎src/common/remote.cpp‎

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -51,10 +51,7 @@ bool Remote::downloadPackage(const Package &d, const String &hash, const path &f
5151
{
5252
return false;
5353
}
54-
// remove first cond when server will be using sfh
55-
if (hash == sha256(fn) || hash == strong_file_hash(fn))
56-
return true;
57-
return false;
54+
return check_file_hash(fn, hash);
5855
};
5956

6057
for (auto &s : primary_sources)

‎src/common/resolver.cpp‎

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -193,7 +193,7 @@ void Resolver::resolve(const Packages &deps, std::function<void()> resolve_actio
193193

194194
void Resolver::download(const DownloadDependency &d, const path &fn)
195195
{
196-
if (!d.remote->downloadPackage(d, d.sha256, fn, query_local_db))
196+
if (!d.remote->downloadPackage(d, d.hash, fn, query_local_db))
197197
{
198198
// if we get hashes from local db
199199
// they can be stalled within server refresh time (15 mins)
@@ -215,7 +215,7 @@ void Resolver::download_and_unpack()
215215
auto &d = dd.second;
216216
auto version_dir = d.getDirSrc();
217217
auto hash_file = d.getStampFilename();
218-
bool must_download = d.getStampHash() != d.sha256 || d.sha256.empty();
218+
bool must_download = d.getStampHash() != d.hash || d.hash.empty();
219219

220220
if (fs::exists(version_dir) && !must_download)
221221
return;
@@ -247,7 +247,7 @@ void Resolver::download_and_unpack()
247247
cleanPackages(d.target_name);
248248

249249
rd.downloads++;
250-
write_file(hash_file, d.sha256);
250+
write_file(hash_file, d.hash);
251251

252252
LOG_INFO(logger, "Unpacking : " << d.target_name << "...");
253253
Files files;
@@ -599,7 +599,10 @@ Resolver::Dependencies getDependenciesFromRemote(const Packages &deps, const Rem
599599
d.ppath = v.first;
600600
d.version = v.second.get<String>("version");
601601
d.flags = decltype(d.flags)(v.second.get<uint64_t>("flags"));
602-
d.sha256 = v.second.get<String>("sha256");
602+
// TODO: remove later sha256 field
603+
d.hash = v.second.get<String>("sha256", "empty_hash");
604+
if (d.hash == "empty_hash")
605+
d.hash = v.second.get<String>("hash", "empty_hash");
603606

604607
if (v.second.find(DEPENDENCIES_NODE) != v.second.not_found())
605608
{

‎src/common/spec.cpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -53,7 +53,7 @@ Specification read_specification(const ptree &p)
5353
get_val(s.package.ppath, "project");
5454
get_val(version, "version");
5555
get_val(s.cppan, "cppan");
56-
get_val(s.sha256, "sha256");
56+
get_val(s.hash, "hash");
5757
get_val(created, "created");
5858

5959
s.package.version = Version(version);

‎src/common/spec.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@ struct Specification
2929
Package package;
3030
Source source;
3131
String cppan;
32-
String sha256;
32+
String hash;
3333
TimePoint created;
3434
};
3535

‎src/support/filesystem.cpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,7 @@ path get_temp_filename(const path &subdir)
4040

4141
String get_stamp_filename(const String &prefix)
4242
{
43-
return prefix + ".sha256";
43+
return prefix + ".hash";
4444
}
4545

4646
String make_archive_name(const String &fn)

‎src/support/hash.cpp‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
/*
2+
* Copyright (C) 2016-2017, Egor Pugin
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
#include "hash.h"
18+
19+
String shorten_hash(const String &data)
20+
{
21+
return shorten_hash(data, CPPAN_CONFIG_HASH_SHORT_LENGTH);
22+
}
23+
24+
String sha256_short(const String &data)
25+
{
26+
return shorten_hash(sha256(data));
27+
}
28+
29+
String hash_config(const String &c)
30+
{
31+
return sha256_short(c);
32+
}
33+
34+
bool check_file_hash(const path &fn, const String &hash)
35+
{
36+
// remove when server will be using strong_file_hash
37+
if (hash == sha256(fn))
38+
return true;
39+
if (hash == strong_file_hash(fn))
40+
return true;
41+
return false;
42+
}

‎src/support/hash.h‎

Lines changed: 4 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -21,17 +21,7 @@
2121
#define CPPAN_CONFIG_HASH_METHOD "SHA256"
2222
#define CPPAN_CONFIG_HASH_SHORT_LENGTH 8
2323

24-
inline String shorten_hash(const String &data)
25-
{
26-
return shorten_hash(data, CPPAN_CONFIG_HASH_SHORT_LENGTH);
27-
}
28-
29-
inline String sha256_short(const String &data)
30-
{
31-
return shorten_hash(sha256(data));
32-
}
33-
34-
inline String hash_config(const String &c)
35-
{
36-
return sha256_short(c);
37-
}
24+
String shorten_hash(const String &data);
25+
String sha256_short(const String &data);
26+
String hash_config(const String &c);
27+
bool check_file_hash(const path &fn, const String &hash);

0 commit comments

Comments
 (0)