Skip to content

Commit

Permalink
Change: Batch all INSERTS into vt_refs
Browse files Browse the repository at this point in the history
  • Loading branch information
@timopollmeier
timopollmeier authored Jun 15, 2023
2 parents 890a314 + c0059fe commit 51d93ce
Show file tree
Hide file tree
Showing 3 changed files with 134 additions and 22 deletions.
10 changes: 9 additions & 1 deletion src/gvmd.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1886,6 +1886,7 @@ gvmd (int argc, char** argv, char *env[])
static gchar *broker_address = NULL;
static gchar *feed_lock_path = NULL;
static int feed_lock_timeout = 0;
static int vt_ref_insert_size = VT_REF_INSERT_SIZE_DEFAULT;
static gchar *vt_verification_collation = NULL;

GString *full_disable_commands = g_string_new ("");
Expand Down Expand Up @@ -2210,6 +2211,11 @@ gvmd (int argc, char** argv, char *env[])
&print_version,
"Print version and exit.",
NULL },
{ "vt-ref-insert-size", '\0', 0, G_OPTION_ARG_INT,
&vt_ref_insert_size,
"Max number of VT refs to insert per statement during VT update,"
" 0 for unlimited, default: "
G_STRINGIFY (VT_REF_INSERT_SIZE_DEFAULT), "<number>" },
{ "vt-verification-collation", '\0', 0, G_OPTION_ARG_STRING,
&vt_verification_collation,
"Set collation for VT verification to <collation>, omit or leave"
Expand Down Expand Up @@ -2293,10 +2299,12 @@ gvmd (int argc, char** argv, char *env[])
/* Set the connection auto retry */
set_scanner_connection_retry (scanner_connection_retry);

/* Set SecInfo update commit size */
/* Set SQL sizes */

set_secinfo_commit_size (secinfo_commit_size);

set_vt_ref_insert_size (vt_ref_insert_size);

/* Set VT verification collation override */
set_vt_verification_collation (vt_verification_collation);

Expand Down
138 changes: 117 additions & 21 deletions src/manage_sql_nvts.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,11 @@ create_tables_nvt (const gchar *);

/* NVT related global options */

/**
* @brief Max number of rows inserted per statement.
*/
static int vt_ref_insert_size = VT_REF_INSERT_SIZE_DEFAULT;

/**
* @brief File socket for OSP NVT update.
*/
Expand Down Expand Up @@ -122,6 +127,20 @@ check_osp_vt_update_socket ()

/* NVT's. */

/**
* @brief Set the VT ref insert size.
*
* @param new_size New size.
*/
void
set_vt_ref_insert_size (int new_size)
{
if (new_size < 0)
vt_ref_insert_size = 0;
else
vt_ref_insert_size = new_size;
}

/**
* @brief Ensures the sanity of nvts cache in DB.
*/
Expand Down Expand Up @@ -263,57 +282,130 @@ find_nvt (const char* oid, nvt_t* nvt)
return FALSE;
}

/**
* @brief SQL batch.
*/
typedef struct
{
GString *sql; ///< SQL buffer.
int max; ///< Max number of inserts.
int size; ///< Number of inserts.
} batch_t;

/**
* @brief Create an SQL batch.
*
* @param[in] max Max number of iterations.
*
* @return Freshly allocated batch.
*/
batch_t *
batch_start (int max)
{
batch_t *b;
b = g_malloc0 (sizeof (batch_t));
b->sql = g_string_new ("");
b->max = max;
return b;
}

/**
* @brief Check an SQL batch.
*
* @para[in] b Batch.
*
* @return 1 init b->str, 0 continue as normal.
*/
int
batch_check (batch_t *b)
{
b->size++;

if (b->size == 1)
// First time, caller must init sql.
return 1;

if (b->max == 0)
return 0;

if (b->size > b->max) {
sql ("%s", b->sql->str);

b->size = 1;

g_string_free (b->sql, TRUE);
b->sql = g_string_new ("");

// Batch just ran, caller must init sql again.
return 1;
}

return 0;
}

/**
* @brief End and free an SQL batch.
*
* @para[in] b Batch.
*/
void
batch_end (batch_t *b)
{
if (b->size > 0) {
g_string_append_printf (b->sql, ";");
sql ("%s", b->sql->str);
}
g_string_free (b->sql, TRUE);
g_free (b);
}

/**
* @brief Insert vt_refs for an NVT.
*
* @param[in] nvti NVT Information.
* @param[in] rebuild True if rebuilding.
* @param[in] batch Batch for inserts.
*/
static void
insert_vt_refs (const nvti_t *nvti, int rebuild)
insert_vt_refs (const nvti_t *nvti, int rebuild, batch_t *batch)
{
int i;
GString *insert;

if (rebuild == 0)
sql ("DELETE FROM vt_refs%s where vt_oid = '%s';",
rebuild ? "_rebuild" : "",
nvti_oid (nvti));

insert = NULL;
for (i = 0; i < nvti_vtref_len (nvti); i++)
{
vtref_t *ref;
gchar *quoted_type, *quoted_id, *quoted_text;
int comma;

comma = 0;
ref = nvti_vtref (nvti, i);
quoted_type = sql_quote (vtref_type (ref));
quoted_id = sql_quote (vtref_id (ref));
quoted_text = sql_quote (vtref_text (ref) ? vtref_text (ref) : "");

if (i == 0) {
insert = g_string_new ("");
g_string_append_printf (insert,
if (batch_check (batch))
g_string_append_printf (batch->sql,
"INSERT into vt_refs%s (vt_oid, type, ref_id, ref_text)"
" VALUES",
rebuild ? "_rebuild" : "");
}
else
comma = 1;

g_string_append_printf (insert,
"%s ('%s', '%s', '%s', '%s')",
i == 0 ? "" : ",",
g_string_append_printf (batch->sql,
// Newline in case it gets logged.
"%s\n ('%s', '%s', '%s', '%s')",
comma ? "," : "",
nvti_oid (nvti), quoted_type, quoted_id, quoted_text);

g_free (quoted_type);
g_free (quoted_id);
g_free (quoted_text);
}

if (insert) {
g_string_append_printf (insert, ";");
sql ("%s", insert->str);
g_string_free (insert, TRUE);
}
}

/**
Expand Down Expand Up @@ -370,11 +462,12 @@ insert_vt_severities (const nvti_t *nvti, int rebuild)
*
* Always called within a transaction.
*
* @param[in] nvti NVT Information.
* @param[in] rebuild True if rebuilding.
* @param[in] nvti NVT Information.
* @param[in] rebuild True if rebuilding.
* @param[in] vt_refs_batch Batch for vt_refs.
*/
static void
insert_nvt (const nvti_t *nvti, int rebuild)
insert_nvt (const nvti_t *nvti, int rebuild, batch_t *vt_refs_batch)
{
gchar *qod_str, *qod_type, *cve;
gchar *quoted_name, *quoted_summary, *quoted_insight, *quoted_affected;
Expand Down Expand Up @@ -424,7 +517,7 @@ insert_nvt (const nvti_t *nvti, int rebuild)
rebuild ? "_rebuild" : "",
nvti_oid (nvti));

insert_vt_refs(nvti, rebuild);
insert_vt_refs (nvti, rebuild, vt_refs_batch);

highest = insert_vt_severities(nvti, rebuild);

Expand Down Expand Up @@ -1598,6 +1691,7 @@ update_nvts_from_vts (element_t *get_vts_response,
int count_modified_vts, count_new_vts;
time_t feed_version_epoch;
char *osp_vt_hash;
batch_t *vt_refs_batch;

count_modified_vts = 0;
count_new_vts = 0;
Expand Down Expand Up @@ -1642,6 +1736,7 @@ update_nvts_from_vts (element_t *get_vts_response,
* To solve both cases, we remove all nvt_preferences. */
sql ("TRUNCATE nvt_preferences;");

vt_refs_batch = batch_start (vt_ref_insert_size);
vt = element_first_child (vts);
while (vt)
{
Expand All @@ -1655,7 +1750,7 @@ update_nvts_from_vts (element_t *get_vts_response,
else
count_modified_vts += 1;

insert_nvt (nvti, rebuild);
insert_nvt (nvti, rebuild, vt_refs_batch);

preferences = NULL;
if (update_preferences_from_vt (vt, nvti_oid (nvti), &preferences))
Expand All @@ -1673,6 +1768,7 @@ update_nvts_from_vts (element_t *get_vts_response,
nvti_free (nvti);
vt = element_next (vt);
}
batch_end (vt_refs_batch);

if (rebuild) {
sql ("DROP VIEW IF EXISTS results_autofp;");
Expand Down
8 changes: 8 additions & 0 deletions src/manage_sql_nvts.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -97,6 +97,14 @@
{ NULL, NULL, KEYWORD_TYPE_UNKNOWN } \
}

/**
* @brief Default for vt_ref_insert_size.
*/
#define VT_REF_INSERT_SIZE_DEFAULT 50000

void
set_vt_ref_insert_size (int);

const char *
get_osp_vt_update_socket ();

Expand Down

0 comments on commit 51d93ce

Please sign in to comment.