fix: properly support default, cds.on.insert and cds.on.update for UPSERT queries

db-service/lib/cqn2sql.js

@@ -142,13 +142,15 @@ class CQN2SQLRenderer {
    */
   CREATE_elements(elements) {
     let sql = ''
+    let keys = ''
     for (let e in elements) {
       const definition = elements[e]
       if (definition.isAssociation) continue
+      if (definition.key) keys = `${keys}, ${this.quote(definition.name)}`
       const s = this.CREATE_element(definition)
-      if (s) sql += `${s}, `
+      if (s) sql += `, ${s}`
     }
-    return sql.slice(0, -2)
+    return `${sql.slice(2)}${keys && `, PRIMARY KEY(${keys.slice(2)})`}`
   }
 
   /**
@@ -491,8 +493,6 @@ class CQN2SQLRenderer {
    */
   INSERT_entries(q) {
     const { INSERT } = q
-    const entity = this.name(q.target?.name || INSERT.into.ref[0])
-    const alias = INSERT.into.as
     const elements = q.elements || q.target?.elements
     if (!elements && !INSERT.entries?.length) {
       return // REVISIT: mtx sends an insert statement without entries and no reference entity
@@ -504,19 +504,14 @@ class CQN2SQLRenderer {
     /** @type {string[]} */
     this.columns = columns
 
+    const alias = INSERT.into.as
+    const entity = this.name(q.target?.name || INSERT.into.ref[0])
     if (!elements) {
       this.entries = INSERT.entries.map(e => columns.map(c => e[c]))
       const param = this.param.bind(this, { ref: ['?'] })
       return (this.sql = `INSERT INTO ${this.quote(entity)}${alias ? ' as ' + this.quote(alias) : ''} (${this.columns.map(c => this.quote(c))}) VALUES (${columns.map(param)})`)
     }
 
-    const extractions = this.managed(
-      columns.map(c => ({ name: c })),
-      elements,
-      !!q.UPSERT,
-    )
-    const extraction = extractions.map(c => c.sql)
-
     // Include this.values for placeholders
     /** @type {unknown[][]} */
     this.entries = []
@@ -530,8 +525,9 @@ class CQN2SQLRenderer {
       this.entries = [[...this.values, stream]]
     }
 
+    const extractions = this._managed = this.managed(columns.map(c => ({ name: c })), elements)
     return (this.sql = `INSERT INTO ${this.quote(entity)}${alias ? ' as ' + this.quote(alias) : ''} (${this.columns.map(c => this.quote(c))
-      }) SELECT ${extraction} FROM json_each(?)`)
+      }) SELECT ${extractions.map(c => c.insert)} FROM json_each(?)`)
   }
 
   async *INSERT_entries_stream(entries, binaryEncoding = 'base64') {
@@ -646,18 +642,7 @@ class CQN2SQLRenderer {
     const entity = this.name(q.target?.name || INSERT.into.ref[0])
     const alias = INSERT.into.as
     const elements = q.elements || q.target?.elements
-    const columns = INSERT.columns
-      || cds.error`Cannot insert rows without columns or elements`
-
-    const inputConverter = this.class._convertInput
-    const extraction = columns.map((c, i) => {
-      const extract = `value->>'$[${i}]'`
-      const element = elements?.[c]
-      const converter = element?.[inputConverter]
-      return converter?.(extract, element) || extract
-    })
-
-    this.columns = columns
+    const columns = this.columns = INSERT.columns || cds.error`Cannot insert rows without columns or elements`
 
     if (!elements) {
       this.entries = INSERT.rows
@@ -675,6 +660,10 @@ class CQN2SQLRenderer {
       this.entries = [[...this.values, stream]]
     }
 
+    const extraction = (this._managed = this.managed(columns.map(c => ({ name: c })), elements))
+      .slice(0, columns.length)
+      .map(c => c.converter(c.extract))
+
     return (this.sql = `INSERT INTO ${this.quote(entity)}${alias ? ' as ' + this.quote(alias) : ''} (${this.columns.map(c => this.quote(c))
       }) SELECT ${extraction} FROM json_each(?)`)
   }
@@ -686,7 +675,7 @@ class CQN2SQLRenderer {
    */
   INSERT_values(q) {
     let { columns, values } = q.INSERT
-    return this.INSERT_rows({ __proto__: q, INSERT: { __proto__: q.INSERT, columns, rows: [values] } })
+    return this.render({ __proto__: q, INSERT: { __proto__: q.INSERT, columns, rows: [values] } })
   }
 
   /**
@@ -737,14 +726,37 @@ class CQN2SQLRenderer {
    */
   UPSERT(q) {
     const { UPSERT } = q
-    const elements = q.target?.elements || {}
+
     let sql = this.INSERT({ __proto__: q, INSERT: UPSERT })
-    let keys = q.target?.keys
-    if (!keys) return this.sql = sql
-    keys = Object.keys(keys).filter(k => !keys[k].isAssociation && !keys[k].virtual)
+    if (!q.target?.keys) return sql
+    const keys = []
+    for (const k of ObjectKeys(q.target?.keys)) {
+      const element = q.target.keys[k]
+      if (element.isAssociation || element.virtual) continue
+      keys.push(k)
+    }
+
+    const elements = q.target?.elements || {}
+    // temporal data
+    for (const k of ObjectKeys(elements)) {
+      if (elements[k]['@cds.valid.from']) keys.push(k)
+    }
 
-    let updateColumns = q.UPSERT.entries ? Object.keys(q.UPSERT.entries[0]) : this.columns
-    updateColumns = updateColumns.filter(c => {
+    const keyCompare = keys
+      .map(k => `NEW.${this.quote(k)}=OLD.${this.quote(k)}`)
+      .join(' AND ')
+
+    const columns = this.columns // this.columns is computed as part of this.INSERT
+    const managed = this._managed.slice(0, columns.length)
+
+    const extractkeys = managed
+      .filter(c => keys.includes(c.name))
+      .map(c => `${c.onInsert || c.sql} as ${this.quote(c.name)}`)
+
+    const entity = this.name(q.target?.name || UPSERT.into.ref[0])
+    sql = `SELECT ${managed.map(c => c.upsert)} FROM (SELECT value, ${extractkeys} from json_each(?)) as NEW LEFT JOIN ${this.quote(entity)} AS OLD ON ${keyCompare}`
+
+    const updateColumns = columns.filter(c => {
       if (keys.includes(c)) return false //> keys go into ON CONFLICT clause
       let e = elements[c]
       if (!e) return true //> pass through to native SQL columns not in CDS model
@@ -754,14 +766,8 @@ class CQN2SQLRenderer {
       else return true
     }).map(c => `${this.quote(c)} = excluded.${this.quote(c)}`)
 
-    // temporal data
-    keys.push(...Object.values(q.target.elements).filter(e => e['@cds.valid.from']).map(e => e.name))
-
-    keys = keys.map(k => this.quote(k))
-    const conflict = updateColumns.length
-      ? `ON CONFLICT(${keys}) DO UPDATE SET ` + updateColumns
-      : `ON CONFLICT(${keys}) DO NOTHING`
-    return (this.sql = `${sql} WHERE true ${conflict}`)
+    return (this.sql = `INSERT INTO ${this.quote(entity)} (${columns.map(c => this.quote(c))}) ${sql
+      } WHERE TRUE ON CONFLICT(${keys.map(c => this.quote(c))}) DO ${updateColumns.length ? `UPDATE SET ${updateColumns}` : 'NOTHING'}`)
   }
 
   // UPDATE Statements ------------------------------------------------
@@ -790,7 +796,9 @@ class CQN2SQLRenderer {
       }
     }
 
-    const extraction = this.managed(columns, elements, true).map(c => `${this.quote(c.name)}=${c.sql}`)
+    const extraction = this.managed(columns, elements)
+      .filter((c, i) => columns[i] || c.onUpdate)
+      .map((c, i) => `${this.quote(c.name)}=${!columns[i] ? c.onUpdate : c.sql}`)
 
     sql += ` SET ${extraction}`
     if (where) sql += ` WHERE ${this.where(where)}`
@@ -1042,56 +1050,104 @@ class CQN2SQLRenderer {
   }
 
   /**
-   * Convers the columns array into an array of SQL expressions that extract the correct value from inserted JSON data
+   * Converts the columns array into an array of SQL expressions that extract the correct value from inserted JSON data
    * @param {object[]} columns
    * @param {import('./infer/cqn').elements} elements
    * @param {Boolean} isUpdate
    * @returns {string[]} Array of SQL expressions for processing input JSON data
    */
-  managed(columns, elements, isUpdate = false) {
-    const annotation = isUpdate ? '@cds.on.update' : '@cds.on.insert'
+  managed(columns, elements) {
+    const cdsOnInsert = '@cds.on.insert'
+    const cdsOnUpdate = '@cds.on.update'
+
     const { _convertInput } = this.class
     // Ensure that missing managed columns are added
     const requiredColumns = !elements
       ? []
-      : Object.keys(elements)
-        .filter(
-          e =>
-            (elements[e]?.[annotation] || (!isUpdate && elements[e]?.default && !elements[e].virtual && !elements[e].isAssociation)) &&
-            !columns.find(c => c.name === e),
-        )
+      : ObjectKeys(elements)
+        .filter(e => {
+          const element = elements[e]
+          // Actual mandatory check
+          if (!(element.default || element[cdsOnInsert] || element[cdsOnUpdate])) return false
+          // Physical column check
+          if (!element || element.virtual || element.isAssociation) return false
+          // Existence check
+          if (columns.find(c => c.name === e)) return false
+          return true
+        })
         .map(name => ({ name, sql: 'NULL' }))
 
+    const keys = ObjectKeys(elements).filter(e => elements[e].key && !elements[e].isAssociation)
+    const keyZero = keys[0] && this.quote(keys[0])
+
     return [...columns, ...requiredColumns].map(({ name, sql }) => {
-      let element = elements?.[name] || {}
-      if (!sql) sql = `value->>'$."${name}"'`
-
-      let converter = element[_convertInput]
-      if (converter && sql[0] !== '$') sql = converter(sql, element)
-
-      let val = _managed[element[annotation]?.['=']]
-      if (val) sql = `coalesce(${sql}, ${this.func({ func: 'session_context', args: [{ val, param: false }] })})`
-      else if (!isUpdate && element.default) {
-        const d = element.default
-        if (d.val !== undefined || d.ref?.[0] === '$now') {
-          // REVISIT: d.ref is not used afterwards
-          sql = `(CASE WHEN json_type(value,'$."${name}"') IS NULL THEN ${this.defaultValue(d.val) // REVISIT: this.defaultValue is a strange function
-            } ELSE ${sql} END)`
-        }
+      const element = elements?.[name] || {}
+
+      const converter = a => element[_convertInput]?.(a, element) || a
+      let extract
+      if (!sql) {
+        ({ sql, extract } = this.managed_extract(name, element, converter))
+      } else {
+        extract = sql = converter(sql)
       }
+      // if (sql[0] !== '$') sql = converter(sql, element)
+
+      let onInsert = this.managed_session_context(element[cdsOnInsert]?.['='])
+        || this.managed_session_context(element.default?.ref?.[0])
+        || (element.default?.val !== undefined && { val: element.default.val, param: false })
+      let onUpdate = this.managed_session_context(element[cdsOnUpdate]?.['='])
+
+      if (onInsert) onInsert = this.expr(onInsert)
+      if (onUpdate) onUpdate = this.expr(onUpdate)
+
+      const qname = this.quote(name)
+
+      const insert = onInsert ? this.managed_default(name, converter(onInsert), sql) : sql
+      const update = onUpdate ? this.managed_default(name, converter(onUpdate), sql) : sql
+      const upsert = keyZero && (
+        // upsert requires the keys to be provided for the existance join (default values optional)
+        element.key
+          // If both insert and update have the same managed definition exclude the old value check
+          || (onInsert && onUpdate && insert === update)
+          ? `${insert} as ${qname}`
+          : `CASE WHEN OLD.${keyZero} IS NULL THEN ${
+          // If key of old is null execute insert
+          insert
+          } ELSE ${
+          // Else execute managed update or keep old if no new data if provided
+          onUpdate ? update : this.managed_default(name, `OLD.${qname}`, update)
+          } END as ${qname}`
+      )
 
-      return { name, sql }
+      return {
+        name, // Element name
+        sql, // Reference SQL
+        extract, // Source SQL
+        converter, // Converter logic
+        // action specific full logic
+        insert, update, upsert,
+        // action specific isolated logic
+        onInsert, onUpdate
+      }
     })
   }
 
-  /**
-   * Returns the default value
-   * @param {string} defaultValue
-   * @returns {string}
-   */
-  // REVISIT: This is a strange method, also overridden inconsistently in postgres
-  defaultValue(defaultValue = this.context.timestamp.toISOString()) {
-    return typeof defaultValue === 'string' ? this.string(defaultValue) : defaultValue
+  managed_extract(name, element, converter) {
+    const { UPSERT, INSERT } = this.cqn
+    const extract = !(INSERT?.entries || UPSERT?.entries) && (INSERT?.rows || UPSERT?.rows)
+      ? `value->>'$[${this.columns.indexOf(name)}]'`
+      : `value->>'$."${name.replace(/"/g, '""')}"'`
+    const sql = converter?.(extract) || extract
+    return { extract, sql }
+  }
+
+  managed_session_context(src) {
+    const val = _managed[src]
+    return val && { func: 'session_context', args: [{ val, param: false }] }
+  }
+
+  managed_default(name, managed, src) {
+    return `(CASE WHEN json_type(value,${this.managed_extract(name).extract.slice(8)}) IS NULL THEN ${managed} ELSE ${src} END)`
   }
 }