github/jooq
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[#5960] [#7094] Duplicate SQL detection

Browse Source 
- [#5960] Duplicate SQL detection
- [#7094] Add ParamType.FORCE_INDEXED
This commit is contained in:
lukaseder 2018-01-23 22:10:18 +01:00
parent 972e330b60
commit b2e4cadf8a
11 changed files with 197 additions and 32 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
jOOQ/src/main/java/org/jooq/DiagnosticsContext.java
View File

@ -38,6 +38,7 @@
package org.jooq;
import java.sql.ResultSet;
import java.util.List;
/**
* A parameter object that is passed to {@link DiagnosticsListener} methods.
@ -90,4 +91,14 @@ public interface DiagnosticsContext {
* {@link #resultSet()}, or <code>-1</code> if there was no result set.
*/
int resultSetActualColumns();
/**
* The normalised statement that all duplicates correspond to.
*/
String normalisedStatement();
/**
* The duplicate statements that all correspond to a single normalised statement.
*/
List<String> duplicateStatements();
}

39
jOOQ/src/main/java/org/jooq/DiagnosticsListener.java
View File

@ -71,4 +71,43 @@ public interface DiagnosticsListener {
*/
void tooManyColumnsFetched(DiagnosticsContext ctx);
/**
* The executed JDBC statement has duplicates.
* <p>
* Many databases maintain an execution plan cache, which remembers
* execution plans for a given SQL string. These caches often use the
* verbatim SQL string (or a hash thereof) as a key, meaning that "similar"
* but not identical statements will produce different keys. This may be
* desired in rare cases when querying skewed data, as a hack to force the
* optimiser to calculate a new plan for a given "similar" but not identical
* query, but mostly, this is not desirable as calculating execution plans
* can turn out to be expensive.
* <p>
* Examples of such similar statements include:
* <p>
* <h3>Whitespace differences</h3>
* <p>
* <code><pre>
* SELECT * FROM actor;
* SELECT * FROM actor;
* </pre></code>
* <p>
* <h3>Inline bind values</h3>
* <p>
* <code><pre>
* SELECT * FROM actor WHERE id = 1;
* SELECT * FROM actor WHERE id = 2;
* </pre></code>
* <p>
* <h3>Aliasing and qualification</h3>
* <p>
* <code><pre>
* SELECT a1.* FROM actor a1 WHERE id = ?;
* SELECT * FROM actor a2 WHERE a2.id = ?;
* </pre></code>
* <p>
* This event is triggered every time a new duplicate is encountered.
*/
void duplicateStatements(DiagnosticsContext ctx);
}

2
jOOQ/src/main/java/org/jooq/conf/ParamType.java
View File

@ -21,6 +21,7 @@ import javax.xml.bind.annotation.XmlType;
* &lt;simpleType name="ParamType"&gt;
* &lt;restriction base="{http://www.w3.org/2001/XMLSchema}string"&gt;
* &lt;enumeration value="INDEXED"/&gt;
* &lt;enumeration value="FORCE_INDEXED"/&gt;
* &lt;enumeration value="NAMED"/&gt;
* &lt;enumeration value="NAMED_OR_INLINED"/&gt;
* &lt;enumeration value="INLINED"/&gt;
@ -34,6 +35,7 @@ import javax.xml.bind.annotation.XmlType;
public enum ParamType {
INDEXED,
FORCE_INDEXED,
NAMED,
NAMED_OR_INLINED,
INLINED;

2
jOOQ/src/main/java/org/jooq/impl/AbstractContext.java
View File

@ -150,6 +150,8 @@ abstract class AbstractContext<C extends Context<C>> extends AbstractScope imple
this.forcedParamType = SettingsTools.getStatementType(settings()) == StatementType.STATIC_STATEMENT
? ParamType.INLINED
: SettingsTools.getParamType(settings()) == ParamType.FORCE_INDEXED
? ParamType.INDEXED
: null;
ParamCastMode m = settings().getParamCastMode();

34
jOOQ/src/main/java/org/jooq/impl/DefaultDiagnosticsContext.java
View File

@ -39,6 +39,9 @@ package org.jooq.impl;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import org.jooq.DiagnosticsContext;
@ -47,11 +50,22 @@ import org.jooq.DiagnosticsContext;
*/
final class DefaultDiagnosticsContext implements DiagnosticsContext {
ResultSet resultSet;
int resultSetFetchedColumns;
int resultSetActualColumns;
int resultSetFetchedRows;
int resultSetActualRows;
ResultSet resultSet;
int resultSetFetchedColumns;
int resultSetActualColumns;
int resultSetFetchedRows;
int resultSetActualRows;
final String normalisedStatement;
final List<String> duplicateStatements;
DefaultDiagnosticsContext(String statement) {
this(statement, Arrays.asList(statement));
}
DefaultDiagnosticsContext(String normalisedStatement, List<String> duplicateStatements) {
this.normalisedStatement = normalisedStatement;
this.duplicateStatements = duplicateStatements;
}
@Override
public final ResultSet resultSet() {
@ -87,4 +101,14 @@ final class DefaultDiagnosticsContext implements DiagnosticsContext {
public final int resultSetActualColumns() {
return resultSet == null ? -1 : resultSetActualColumns;
}
@Override
public final String normalisedStatement() {
return normalisedStatement;
}
@Override
public final List<String> duplicateStatements() {
return Collections.unmodifiableList(duplicateStatements);
}
}

6
jOOQ/src/main/java/org/jooq/impl/DefaultDiagnosticsListener.java
View File

@ -53,4 +53,10 @@ public class DefaultDiagnosticsListener implements DiagnosticsListener {
@Override
public void tooManyRowsFetched(DiagnosticsContext ctx) {}
@Override
public void tooManyColumnsFetched(DiagnosticsContext ctx) {}
@Override
public void duplicateStatements(DiagnosticsContext ctx) {}
}

92
jOOQ/src/main/java/org/jooq/impl/DiagnosticsConnection.java
View File

@ -37,12 +37,26 @@
*/
package org.jooq.impl;
import static org.jooq.conf.ParamType.FORCE_INDEXED;
import java.sql.CallableStatement;
import java.sql.PreparedStatement;
import java.sql.SQLException;
import java.sql.Statement;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import org.jooq.Configuration;
import org.jooq.Parser;
import org.jooq.Queries;
import org.jooq.RenderContext;
import org.jooq.conf.SettingsTools;
import org.jooq.tools.jdbc.DefaultConnection;
/**
@ -50,13 +64,19 @@ import org.jooq.tools.jdbc.DefaultConnection;
*/
final class DiagnosticsConnection extends DefaultConnection {
final Configuration configuration;
final DiagnosticsListeners listeners;
static final Map<String, Set<String>> DUPLICATE_SQL = Collections.synchronizedMap(new LRU());
final Configuration configuration;
final RenderContext forceIndexed;
final Parser parser;
final DiagnosticsListeners listeners;
@SuppressWarnings("deprecation")
DiagnosticsConnection(Configuration configuration) {
super(configuration.connectionProvider().acquire());
this.configuration = configuration;
this.forceIndexed = configuration.derive(SettingsTools.clone(configuration.settings()).withParamType(FORCE_INDEXED)).dsl().renderContext();
this.parser = configuration.dsl().parser();
this.listeners = DiagnosticsListeners.get(configuration);
}
@ -77,51 +97,101 @@ final class DiagnosticsConnection extends DefaultConnection {
@Override
public final PreparedStatement prepareStatement(String sql) throws SQLException {
return new DiagnosticsStatement(this, getDelegate().prepareStatement(sql));
return new DiagnosticsStatement(this, getDelegate().prepareStatement(parse(sql)));
}
@Override
public final PreparedStatement prepareStatement(String sql, int resultSetType, int resultSetConcurrency) throws SQLException {
return new DiagnosticsStatement(this, getDelegate().prepareStatement(sql, resultSetType, resultSetConcurrency));
return new DiagnosticsStatement(this, getDelegate().prepareStatement(parse(sql), resultSetType, resultSetConcurrency));
}
@Override
public final PreparedStatement prepareStatement(String sql, int resultSetType, int resultSetConcurrency, int resultSetHoldability) throws SQLException {
return new DiagnosticsStatement(this, getDelegate().prepareStatement(sql, resultSetType, resultSetConcurrency, resultSetHoldability));
return new DiagnosticsStatement(this, getDelegate().prepareStatement(parse(sql), resultSetType, resultSetConcurrency, resultSetHoldability));
}
@Override
public final PreparedStatement prepareStatement(String sql, int autoGeneratedKeys) throws SQLException {
return new DiagnosticsStatement(this, getDelegate().prepareStatement(sql, autoGeneratedKeys));
return new DiagnosticsStatement(this, getDelegate().prepareStatement(parse(sql), autoGeneratedKeys));
}
@Override
public final PreparedStatement prepareStatement(String sql, int[] columnIndexes) throws SQLException {
return new DiagnosticsStatement(this, getDelegate().prepareStatement(sql, columnIndexes));
return new DiagnosticsStatement(this, getDelegate().prepareStatement(parse(sql), columnIndexes));
}
@Override
public final PreparedStatement prepareStatement(String sql, String[] columnNames) throws SQLException {
return new DiagnosticsStatement(this, getDelegate().prepareStatement(sql, columnNames));
return new DiagnosticsStatement(this, getDelegate().prepareStatement(parse(sql), columnNames));
}
@Override
public final CallableStatement prepareCall(String sql) throws SQLException {
return new DiagnosticsStatement(this, getDelegate().prepareCall(sql));
return new DiagnosticsStatement(this, getDelegate().prepareCall(parse(sql)));
}
@Override
public final CallableStatement prepareCall(String sql, int resultSetType, int resultSetConcurrency) throws SQLException {
return new DiagnosticsStatement(this, getDelegate().prepareCall(sql, resultSetType, resultSetConcurrency));
return new DiagnosticsStatement(this, getDelegate().prepareCall(parse(sql), resultSetType, resultSetConcurrency));
}
@Override
public final CallableStatement prepareCall(String sql, int resultSetType, int resultSetConcurrency, int resultSetHoldability) throws SQLException {
return new DiagnosticsStatement(this, getDelegate().prepareCall(sql, resultSetType, resultSetConcurrency, resultSetHoldability));
return new DiagnosticsStatement(this, getDelegate().prepareCall(parse(sql), resultSetType, resultSetConcurrency, resultSetHoldability));
}
@Override
public final void close() throws SQLException {
configuration.connectionProvider().release(getDelegate());
}
@SuppressWarnings("deprecation")
final String parse(String sql) {
Queries queries;
try {
queries = parser.parse(sql);
}
catch (ParserException ignore) {
return sql;
}
String normalised = forceIndexed.render(queries);
List<String> duplicates = null;
synchronized (DUPLICATE_SQL) {
Set<String> v = DUPLICATE_SQL.get(normalised);
if (v == null) {
v = new HashSet<String>();
DUPLICATE_SQL.put(normalised, v);
}
if (v.size() >= DUP_SIZE || (v.add(sql) && v.size() > 1))
duplicates = new ArrayList<String>(v);
}
if (duplicates != null)
listeners.duplicateStatements(new DefaultDiagnosticsContext(normalised, duplicates));
return sql;
}
// TODO: Make this configurable
static final int LRU_SIZE = 50000;
static final int DUP_SIZE = 500;
// See https://stackoverflow.com/a/1953516/521799
static class LRU extends LinkedHashMap<String, Set<String>> {
private static final long serialVersionUID = 5287799057535876982L;
LRU() {
super(LRU_SIZE + 1, 1.0f, true);
}
@Override
protected boolean removeEldestEntry(Entry<String, Set<String>> eldest) {
return size() > LRU_SIZE;
}
}
}

6
jOOQ/src/main/java/org/jooq/impl/DiagnosticsListeners.java
View File

@ -72,4 +72,10 @@ final class DiagnosticsListeners implements DiagnosticsListener {
for (DiagnosticsListener listener : listeners)
listener.tooManyColumnsFetched(ctx);
}
@Override
public final void duplicateStatements(DiagnosticsContext ctx) {
for (DiagnosticsListener listener : listeners)
listener.duplicateStatements(ctx);
}
}

6
jOOQ/src/main/java/org/jooq/impl/DiagnosticsResultSet.java
View File

@ -67,16 +67,18 @@ import org.jooq.tools.jdbc.DefaultResultSet;
final class DiagnosticsResultSet extends DefaultResultSet {
final DiagnosticsConnection connection;
final String sql;
final ResultSetMetaData meta;
final BitSet read;
final int columns;
int current;
int rows;
DiagnosticsResultSet(ResultSet delegate, Statement creator, DiagnosticsConnection connection) throws SQLException {
DiagnosticsResultSet(ResultSet delegate, String sql, Statement creator, DiagnosticsConnection connection) throws SQLException {
super(delegate, creator);
this.connection = connection;
this.sql = sql;
this.meta = delegate.getMetaData();
this.columns = meta.getColumnCount();
this.read = new BitSet(columns);
@ -590,7 +592,7 @@ final class DiagnosticsResultSet extends DefaultResultSet {
}
private final DefaultDiagnosticsContext ctx() {
DefaultDiagnosticsContext ctx = new DefaultDiagnosticsContext();
DefaultDiagnosticsContext ctx = new DefaultDiagnosticsContext(sql);
ctx.resultSet = super.getDelegate();
ctx.resultSetFetchedColumns = read.cardinality();

28
jOOQ/src/main/java/org/jooq/impl/DiagnosticsStatement.java
View File

@ -59,76 +59,76 @@ final class DiagnosticsStatement extends DefaultCallableStatement {
@Override
public final ResultSet executeQuery(String sql) throws SQLException {
return new DiagnosticsResultSet(super.executeQuery(sql), this, connection);
return new DiagnosticsResultSet(super.executeQuery(connection.parse(sql)), sql, this, connection);
}
@Override
public final int executeUpdate(String sql) throws SQLException {
return super.executeUpdate(sql);
return super.executeUpdate(connection.parse(sql));
}
@Override
public final int executeUpdate(String sql, int autoGeneratedKeys) throws SQLException {
return super.executeUpdate(sql, autoGeneratedKeys);
return super.executeUpdate(connection.parse(sql), autoGeneratedKeys);
}
@Override
public final int executeUpdate(String sql, int[] columnIndexes) throws SQLException {
return super.executeUpdate(sql, columnIndexes);
return super.executeUpdate(connection.parse(sql), columnIndexes);
}
@Override
public final int executeUpdate(String sql, String[] columnNames) throws SQLException {
return super.executeUpdate(sql, columnNames);
return super.executeUpdate(connection.parse(sql), columnNames);
}
@Override
public final boolean execute(String sql) throws SQLException {
return super.execute(sql);
return super.execute(connection.parse(sql));
}
@Override
public final boolean execute(String sql, int autoGeneratedKeys) throws SQLException {
return super.execute(sql, autoGeneratedKeys);
return super.execute(connection.parse(sql), autoGeneratedKeys);
}
@Override
public final boolean execute(String sql, int[] columnIndexes) throws SQLException {
return super.execute(sql, columnIndexes);
return super.execute(connection.parse(sql), columnIndexes);
}
@Override
public final boolean execute(String sql, String[] columnNames) throws SQLException {
return super.execute(sql, columnNames);
return super.execute(connection.parse(sql), columnNames);
}
@Override
public final long executeLargeUpdate(String sql) throws SQLException {
return super.executeLargeUpdate(sql);
return super.executeLargeUpdate(connection.parse(sql));
}
@Override
public final long executeLargeUpdate(String sql, int autoGeneratedKeys) throws SQLException {
return super.executeLargeUpdate(sql, autoGeneratedKeys);
return super.executeLargeUpdate(connection.parse(sql), autoGeneratedKeys);
}
@Override
public final long executeLargeUpdate(String sql, int[] columnIndexes) throws SQLException {
return super.executeLargeUpdate(sql, columnIndexes);
return super.executeLargeUpdate(connection.parse(sql), columnIndexes);
}
@Override
public final long executeLargeUpdate(String sql, String[] columnNames) throws SQLException {
return super.executeLargeUpdate(sql, columnNames);
return super.executeLargeUpdate(connection.parse(sql), columnNames);
}
@Override
public final void addBatch(String sql) throws SQLException {
super.addBatch(sql);
super.addBatch(connection.parse(sql));
}
@Override

3
jOOQ/src/main/resources/xsd/jooq-runtime-3.11.0.xsd
View File

@ -278,6 +278,9 @@ Either &lt;input/> or &lt;inputExpression/> must be provided]]></jxb:javadoc></j
<!-- Execute statements with indexed parameters, the way JDBC expects them -->
<enumeration value="INDEXED"/>
<!-- Execute statements with indexed parameters, forcing explicit inlined and named parameters to be indexed as well -->
<enumeration value="FORCE_INDEXED"/>
<!-- Execute statements with named parameters -->
<enumeration value="NAMED"/>