bigboss/StatisticDatabaseGeneratorAlgorithms
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

initial commit

Browse Source
This commit is contained in:
tom
2026-06-03 23:24:12 +02:00
commit accd42068d
152 changed files with 100216 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
DatabaseGenerator/src/Database.hpp
+246
View File
@@ -0,0 +1,246 @@
#pragma once
#include <sqlite3.h>
#include <cstdint>
#include <stdexcept>
#include <string>
#include <functional>
#include <vector>
#include <optional>
#include <variant>
#include <map>
// ─────────────────────────────────────────────
// Exceptions
// ─────────────────────────────────────────────
struct DbError : std::runtime_error {
int code;
DbError(int rc, const std::string& msg)
: std::runtime_error(msg), code(rc) {}
};
// ─────────────────────────────────────────────
// Value : type SQL générique
// ─────────────────────────────────────────────
using DbNull = std::monostate;
using DbBlob = std::vector<uint8_t>;
using DbValue = std::variant<DbNull, int64_t, double, std::string, DbBlob>;
inline DbValue db_null() { return DbNull{}; }
inline DbValue db_int (int64_t v) { return v; }
inline DbValue db_real(double v) { return v; }
inline DbValue db_text(const std::string& v) { return v; }
inline DbValue db_blob(const DbBlob& v) { return v; }
// ─────────────────────────────────────────────
// ResultRow : ligne copiée depuis un statement
// (valeurs extraites, safe après sqlite3_reset)
// ─────────────────────────────────────────────
struct ResultRow {
std::vector<std::string> names;
std::vector<DbValue> values;
int size() const { return static_cast<int>(values.size()); }
const DbValue& get(int col) const { return values.at(col); }
const DbValue& get(const std::string& name) const {
for (int i = 0; i < static_cast<int>(names.size()); ++i)
if (names[i] == name) return values[i];
throw std::out_of_range("Column not found: " + name);
}
int64_t get_int (int col, int64_t def = 0) const { auto& v = get(col); return std::holds_alternative<int64_t>(v) ? std::get<int64_t>(v) : def; }
double get_real(int col, double def = 0.0) const { auto& v = get(col); return std::holds_alternative<double>(v) ? std::get<double>(v) : def; }
std::string get_text(int col, std::string def = {}) const { auto& v = get(col); return std::holds_alternative<std::string>(v) ? std::get<std::string>(v) : def; }
bool is_null (int col) const { return std::holds_alternative<DbNull>(get(col)); }
int64_t get_int (const std::string& n, int64_t def = 0) const { return get_int (col_idx(n), def); }
double get_real(const std::string& n, double def = 0.0) const { return get_real(col_idx(n), def); }
std::string get_text(const std::string& n, std::string def = {}) const { return get_text(col_idx(n), def); }
bool is_null (const std::string& n) const { return is_null (col_idx(n)); }
private:
int col_idx(const std::string& name) const {
for (int i = 0; i < static_cast<int>(names.size()); ++i)
if (names[i] == name) return i;
throw std::out_of_range("Column not found: " + name);
}
};
// Helper interne : copie une ligne depuis un stmt actif
inline ResultRow snapshot_row(sqlite3_stmt* stmt) {
ResultRow row;
int n = sqlite3_column_count(stmt);
row.names.reserve(n);
row.values.reserve(n);
for (int col = 0; col < n; ++col) {
row.names.push_back(sqlite3_column_name(stmt, col));
switch (sqlite3_column_type(stmt, col)) {
case SQLITE_INTEGER: row.values.push_back(static_cast<int64_t>(sqlite3_column_int64(stmt, col))); break;
case SQLITE_FLOAT: row.values.push_back(sqlite3_column_double(stmt, col)); break;
case SQLITE_TEXT: {
auto* p = sqlite3_column_text(stmt, col);
row.values.push_back(std::string(reinterpret_cast<const char*>(p)));
break;
}
case SQLITE_BLOB: {
auto* p = static_cast<const uint8_t*>(sqlite3_column_blob(stmt, col));
int nb = sqlite3_column_bytes(stmt, col);
row.values.push_back(DbBlob(p, p + nb));
break;
}
default: row.values.push_back(DbNull{}); break;
}
}
return row;
}
// ─────────────────────────────────────────────
// Statement : requête préparée réutilisable
// ─────────────────────────────────────────────
class Statement {
public:
Statement() = default;
Statement(const Statement&) = delete;
Statement& operator=(const Statement&) = delete;
Statement(Statement&& o) noexcept : stmt_(o.stmt_) { o.stmt_ = nullptr; }
Statement& operator=(Statement&& o) noexcept {
finalize(); stmt_ = o.stmt_; o.stmt_ = nullptr; return *this;
}
~Statement() { finalize(); }
bool valid() const { return stmt_ != nullptr; }
// ── Bind par index (1-based) ──────────────────────────────
Statement& bind(int i, DbNull) { sqlite3_bind_null (stmt_, i); return *this; }
Statement& bind(int i, int64_t v) { sqlite3_bind_int64 (stmt_, i, v); return *this; }
Statement& bind(int i, int v) { return bind(i, static_cast<int64_t>(v)); }
Statement& bind(int i, double v) { sqlite3_bind_double(stmt_, i, v); return *this; }
Statement& bind(int i, const std::string& v) { sqlite3_bind_text (stmt_, i, v.c_str(), -1, SQLITE_TRANSIENT); return *this; }
Statement& bind(int i, const char* v) { sqlite3_bind_text (stmt_, i, v, -1, SQLITE_TRANSIENT); return *this; }
Statement& bind(int i, const DbBlob& v) { sqlite3_bind_blob (stmt_, i, v.data(), static_cast<int>(v.size()), SQLITE_TRANSIENT); return *this; }
Statement& bind(int i, const DbValue& v) { std::visit([&](auto&& x){ this->bind(i, x); }, v); return *this; }
// ── Bind par nom (:nom, $nom, @nom) ──────────────────────
template<typename T>
Statement& bind(const std::string& name, T&& v) {
return bind(param_index(name), std::forward<T>(v));
}
// ── Exécution ────────────────────────────────────────────
// Itère sur chaque ligne copiée (safe, row valide après reset)
int each(std::function<void(const ResultRow&)> cb) {
int count = 0;
while (true) {
int rc = sqlite3_step(stmt_);
if (rc == SQLITE_ROW) { cb(snapshot_row(stmt_)); ++count; }
else if (rc == SQLITE_DONE) { break; }
else { sqlite3_reset(stmt_); throw DbError(rc, sqlite3_errmsg(sqlite3_db_handle(stmt_))); }
}
sqlite3_reset(stmt_);
return count;
}
// Première ligne (ou nullopt)
std::optional<ResultRow> first() {
std::optional<ResultRow> result;
int rc = sqlite3_step(stmt_);
if (rc == SQLITE_ROW) result = snapshot_row(stmt_);
sqlite3_reset(stmt_);
if (rc != SQLITE_ROW && rc != SQLITE_DONE)
throw DbError(rc, sqlite3_errmsg(sqlite3_db_handle(stmt_)));
return result;
}
// INSERT / UPDATE / DELETE — renvoie le nb de lignes affectées
int exec() {
int rc = sqlite3_step(stmt_);
sqlite3_reset(stmt_);
if (rc != SQLITE_DONE && rc != SQLITE_ROW)
throw DbError(rc, sqlite3_errmsg(sqlite3_db_handle(stmt_)));
return sqlite3_changes(sqlite3_db_handle(stmt_));
}
Statement& clear_bindings() { sqlite3_clear_bindings(stmt_); return *this; }
private:
friend class Database;
sqlite3_stmt* stmt_ = nullptr;
explicit Statement(sqlite3_stmt* s) : stmt_(s) {}
void finalize() { if (stmt_) { sqlite3_finalize(stmt_); stmt_ = nullptr; } }
int param_index(const std::string& name) const {
int i = sqlite3_bind_parameter_index(stmt_, name.c_str());
if (i == 0) throw std::out_of_range("Parameter not found: " + name);
return i;
}
};
// ─────────────────────────────────────────────
// Database : gestion de la connexion
// ─────────────────────────────────────────────
class Database {
public:
explicit Database(const std::string& path,
int flags = SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE) {
int rc = sqlite3_open_v2(path.c_str(), &db_, flags, nullptr);
if (rc != SQLITE_OK) {
std::string msg = db_ ? sqlite3_errmsg(db_) : "sqlite3_open_v2 failed";
sqlite3_close(db_);
throw DbError(rc, msg);
}
exec("PRAGMA journal_mode=WAL");
exec("PRAGMA foreign_keys=ON");
}
~Database() { if (db_) sqlite3_close(db_); }
Database(const Database&) = delete;
Database& operator=(const Database&) = delete;
Database(Database&& o) noexcept : db_(o.db_) { o.db_ = nullptr; }
// ── Requête préparée ──────────────────────────────────────
Statement prepare(const std::string& sql) {
sqlite3_stmt* stmt = nullptr;
int rc = sqlite3_prepare_v2(db_, sql.c_str(), -1, &stmt, nullptr);
if (rc != SQLITE_OK)
throw DbError(rc, sqlite3_errmsg(db_));
return Statement(stmt);
}
// ── Exécution directe (sans résultat) ────────────────────
void exec(const std::string& sql) {
char* err = nullptr;
int rc = sqlite3_exec(db_, sql.c_str(), nullptr, nullptr, &err);
if (rc != SQLITE_OK) {
std::string msg = err ? err : "exec failed";
sqlite3_free(err);
throw DbError(rc, msg);
}
}
// ── Transaction RAII ─────────────────────────────────────
struct Transaction {
Database& db;
bool done = false;
explicit Transaction(Database& db) : db(db) { db.exec("BEGIN"); }
~Transaction() { if (!done) db.exec("ROLLBACK"); }
void commit() { db.exec("COMMIT"); done = true; }
void rollback() { db.exec("ROLLBACK"); done = true; }
};
Transaction begin() { return Transaction(*this); }
// ── Utilitaires ──────────────────────────────────────────
int64_t last_insert_rowid() const { return sqlite3_last_insert_rowid(db_); }
int changes() const { return sqlite3_changes(db_); }
sqlite3* handle() { return db_; }
private:
sqlite3* db_ = nullptr;
};
DatabaseGenerator/src/main.cpp
+262
View File
@@ -0,0 +1,262 @@
#include "Database.hpp"
#include <nlohmann/json.hpp>
#include <filesystem>
#include <fstream>
#include <iostream>
#include <string>
#include <vector>
#include <stdexcept>
namespace fs = std::filesystem;
using json = nlohmann::json;
// ─────────────────────────────────────────────────────────────
// Structure : nom d'instance décomposé depuis le nom de fichier
// Format : Site_SiteCr_Tracks_Rames_Jobs_ID
// ─────────────────────────────────────────────────────────────
struct InstanceKey {
int64_t site;
int64_t site_cr;
int64_t tracks;
int64_t rames;
int64_t jobs;
int64_t id;
std::string raw; // nom complet sans extension
};
InstanceKey parse_instance_name(const std::string& stem) {
InstanceKey k;
k.raw = stem;
std::vector<int64_t> parts;
std::string token;
for (char c : stem) {
if (c == '_') {
if (!token.empty()) { parts.push_back(std::stoll(token)); token.clear(); }
} else {
token += c;
}
}
if (!token.empty()) parts.push_back(std::stoll(token));
if (parts.size() != 6)
throw std::runtime_error("Nom d'instance invalide (attendu 6 entiers) : " + stem);
k.site = parts[0];
k.site_cr = parts[1];
k.tracks = parts[2];
k.rames = parts[3];
k.jobs = parts[4];
k.id = parts[5];
return k;
}
// ─────────────────────────────────────────────────────────────
// Création des tables
// ─────────────────────────────────────────────────────────────
void create_tables(Database& db) {
db.exec(R"(
CREATE TABLE IF NOT EXISTS instances (
instance_id INTEGER PRIMARY KEY AUTOINCREMENT,
name TEXT NOT NULL UNIQUE,
site INTEGER NOT NULL,
site_cr INTEGER NOT NULL,
tracks INTEGER NOT NULL,
rames INTEGER NOT NULL,
jobs INTEGER NOT NULL,
id INTEGER NOT NULL
)
)");
db.exec(R"(
CREATE TABLE IF NOT EXISTS results (
result_id INTEGER PRIMARY KEY AUTOINCREMENT,
instance_id INTEGER NOT NULL REFERENCES instances(instance_id),
algo TEXT NOT NULL,
fct_objectif INTEGER NOT NULL,
status TEXT NOT NULL,
time_final REAL NOT NULL,
best_alg TEXT,
UNIQUE(instance_id, algo)
)
)");
}
// ─────────────────────────────────────────────────────────────
// Insertion / récupération d'une instance (upsert)
// ─────────────────────────────────────────────────────────────
int64_t upsert_instance(Database& /*db*/,
Statement& ins_inst,
Statement& sel_inst,
const InstanceKey& k) {
ins_inst.bind(":name", k.raw)
.bind(":site", k.site)
.bind(":site_cr", k.site_cr)
.bind(":tracks", k.tracks)
.bind(":rames", k.rames)
.bind(":jobs", k.jobs)
.bind(":id", k.id)
.exec();
auto row = sel_inst.bind(":name", k.raw).first();
if (!row) throw std::runtime_error("Instance introuvable après insertion : " + k.raw);
return row->get_int("instance_id");
}
// ─────────────────────────────────────────────────────────────
// Parse un fichier JSON et insère les résultats
// ─────────────────────────────────────────────────────────────
void process_file(const fs::path& path,
Database& db,
Statement& ins_inst,
Statement& sel_inst,
Statement& ins_result) {
// -- Nom d'instance depuis le stem du fichier
InstanceKey key = parse_instance_name(path.stem().string());
int64_t inst_id = upsert_instance(db, ins_inst, sel_inst, key);
// -- Lecture JSON
std::ifstream f(path);
if (!f) throw std::runtime_error("Impossible d'ouvrir : " + path.string());
json doc;
try { f >> doc; }
catch (const json::exception& e) {
throw std::runtime_error("JSON invalide dans " + path.string() + " : " + e.what());
}
if (!doc.is_object())
throw std::runtime_error("Le JSON doit être un objet à la racine : " + path.string());
// -- Chaque clé = un algo
for (auto& [algo_name, obj] : doc.items()) {
if (!obj.is_object()) continue;
if(obj.contains("fctObjectif"))
{
int64_t fct_obj = obj.value("fctObjectif", int64_t{0});
std::string status = obj.value("status", std::string{});
double time_f = obj.value("timeFinal", 0.0);
std::string best_alg = obj.value("bestAlg", algo_name);
ins_result.bind(":instance_id", inst_id)
.bind(":algo", algo_name)
.bind(":fct_objectif", fct_obj)
.bind(":status", status)
.bind(":time_final", time_f)
.bind(":best_alg", best_alg)
.exec();
}
else {
int64_t fct_obj = obj.value("obj", int64_t{0});
std::string status = "Heuristic";
double time_f = obj.value("timeFinal", 0.0);
std::string best_alg = obj.value("source", std::string{});
ins_result.bind(":instance_id", inst_id)
.bind(":algo", algo_name)
.bind(":fct_objectif", fct_obj)
.bind(":status", status)
.bind(":time_final", time_f)
.bind(":best_alg", best_alg)
.exec();
}
}
}
// ─────────────────────────────────────────────────────────────
// Collecte récursive des .json dans un dossier
// ─────────────────────────────────────────────────────────────
std::vector<fs::path> collect_json_files(const fs::path& dir) {
std::vector<fs::path> files;
if (!fs::is_directory(dir)) {
std::cerr << "[WARN] Pas un dossier, ignoré : " << dir << "\n";
return files;
}
for (auto& entry : fs::recursive_directory_iterator(dir)) {
if (entry.is_regular_file() && entry.path().extension() == ".json")
files.push_back(entry.path());
}
return files;
}
// ─────────────────────────────────────────────────────────────
// Main
// ─────────────────────────────────────────────────────────────
int main(int argc, char* argv[]) {
if (argc < 3) {
std::cerr << "Usage: " << argv[0] << " <resultats.db> <dossier1> [dossier2 ...]\n";
return 1;
}
const std::string db_path = argv[1];
std::vector<fs::path> dirs;
for (int i = 2; i < argc; ++i) dirs.emplace_back(argv[i]);
// -- Collecte des fichiers
std::vector<fs::path> all_files;
for (auto& d : dirs) {
auto files = collect_json_files(d);
all_files.insert(all_files.end(), files.begin(), files.end());
}
if (all_files.empty()) {
std::cerr << "[INFO] Aucun fichier .json trouvé.\n";
return 0;
}
std::cout << "[INFO] " << all_files.size() << " fichier(s) trouvé(s).\n";
try {
Database db(db_path);
create_tables(db);
// Requêtes préparées — compilées une seule fois
auto ins_inst = db.prepare(R"(
INSERT OR IGNORE INTO instances (name, site, site_cr, tracks, rames, jobs, id)
VALUES (:name, :site, :site_cr, :tracks, :rames, :jobs, :id)
)");
auto sel_inst = db.prepare(
"SELECT instance_id FROM instances WHERE name = :name"
);
auto ins_result = db.prepare(R"(
INSERT OR REPLACE INTO results
(instance_id, algo, fct_objectif, status, time_final, best_alg)
VALUES
(:instance_id, :algo, :fct_objectif, :status, :time_final, :best_alg)
)");
int ok = 0, err = 0;
// Une transaction par lot = bien plus rapide qu'une transaction par fichier
auto tx = db.begin();
for (auto& path : all_files) {
try {
process_file(path, db, ins_inst, sel_inst, ins_result);
++ok;
if (ok % 100 == 0)
std::cout << " [" << ok << "/" << all_files.size() << "]\n";
} catch (const std::exception& e) {
std::cerr << "[ERR] " << path.filename() << " : " << e.what() << "\n";
++err;
}
}
tx.commit();
std::cout << "[DONE] " << ok << " ok, " << err << " erreur(s).\n";
} catch (const DbError& e) {
std::cerr << "[DB ERR " << e.code << "] " << e.what() << "\n";
return 1;
} catch (const std::exception& e) {
std::cerr << "[ERR] " << e.what() << "\n";
return 1;
}
return 0;
}