myBATIS foreach 达到 1000 的限制

Question

这里是what myBATIS has on their own documentation for foreach。

<select id="selectPostIn" resultType="domain.blog.Post">
  SELECT *
  FROM POST P
  WHERE ID in
  <foreach item="item" index="index" collection="list"
      open="(" separator="," close=")">
        #{item}
  </foreach>
</select>

但是，如果 list 包含超过 1000 个项目并且您使用的是 Oracle DB，则会出现此异常：

java.sql.SQLSyntaxErrorException: ORA-01795: maximum number of expressions in a list is 1000

我该怎么做才能解决此问题，使其适用于超过 1000 个元素？

Answer 1

我不确定这是否是最优雅的解决方案，但这是我所做的：

<select id="selectPostIn" resultType="domain.blog.Post">
  SELECT *
  FROM POST P
  WHERE ID in
  <trim suffixOverrides=" OR ID IN ()">
      <foreach item="item" index="index" collection="list"
               open="(" close=")">
          <if test="index != 0">
              <choose>
                  <when test="index % 1000 == 999">) OR ID IN (</when>
                  <otherwise>,</otherwise>
              </choose>
          </if>
          #{item}
      </foreach>
  </trim>
</select>

说明

让我们从 foreach 开始。我们想把它包围在 ( 和 ) 中。我们希望在大多数元素之间使用逗号，除了我们希望停止列表的每千个元素和 OR 与另一个元素。这就是 choose、when、otherwise 结构处理的内容。除了我们不想要第一个元素之前的任何一个，因此 choose 位于其中的 if 。最后，foreach 以实际插入 #{item} 结束。

外层 trim 只是为了让我们恰好有 1000 个元素，例如，我们不会以 OR ID IN () 结尾，这将是无效的（()，具体来说，是无效部分。这是 SQL 中的语法错误，而不是我希望的空列表。）

Answer 2

我们已经尝试在超过 1000 条记录的子句中使用上述参考删除查询：

 <delete id="delete" parameterType="Map">

以下查询有效：

DELETE FROM Employee
where
 emp_id = #{empId}
  <foreach item="deptId" index= "index" collection="ids" open="AND DEPT_ID NOT IN (" close=")" >
      <if test="index != 0">
        <choose>
                    <when test="index % 1000 == 999">) AND DEPT_ID NOT IN (</when>
                    <otherwise>,</otherwise>
                </choose>
           </if>
  #{deptId}
</foreach>
</delete>

Answer 3

Mybatis 插件查询，然后合并分区参数：

@Intercepts({
    @Signature(type = Executor.class, method = "query", args = {MappedStatement.class, Object.class, RowBounds.class, ResultHandler.class}),
    @Signature(type = Executor.class, method = "query", args = {MappedStatement.class, Object.class, RowBounds.class, ResultHandler.class, CacheKey.class, BoundSql.class})}
)
public class BigSizeParamQueryPlugin implements Interceptor {

private final int singleBatchSize;
private static final HeavyParamContext NO_BIG_PARAM = new HeavyParamContext();

public BigSizeParamQueryPlugin() {
    this.singleBatchSize = 1000;
}

public BigSizeParamQueryPlugin(Integer singleBatchSize) {
    if (singleBatchSize < 500) {
        throw new IllegalArgumentException("batch size less than 500 is not recommended");
    }
    this.singleBatchSize = singleBatchSize;
}

@Override
public Object intercept(Invocation invocation) throws Throwable {

    Object[] args = invocation.getArgs();
    Object parameter = args[1];

    if (parameter instanceof MapperMethod.ParamMap && RowBounds.DEFAULT == args[2]) {
        MapperMethod.ParamMap paramMap = (MapperMethod.ParamMap) parameter;
        if (MapUtils.isNotEmpty(paramMap)) {
            try {
                HeavyParamContext context = findHeavyParam(paramMap);
                if (context.hasHeavyParam()) {
                    QueryExecutor queryExecutor = new QueryExecutor(invocation, context);
                    return queryExecutor.query();
                }
            } catch (Throwable e) {
                log.warn("BigSizeParamQueryPlugin process error", e);
                return invocation.proceed();
            }
        }
    }

    return invocation.proceed();
}

private class QueryExecutor {
    private final MappedStatement ms;
    private final Map<String, Object> paramMap;
    private final RowBounds rowBounds;
    private final ResultHandler resultHandler;
    private final Executor executor;
    private final List<Object> finalResult;
    private final Iterator<HeavyParam> heavyKeyIter;

    public QueryExecutor(Invocation invocation, HeavyParamContext context) {
        Object[] args = invocation.getArgs();
        this.ms = (MappedStatement) args[0];
        this.paramMap = context.getParameter();
        this.rowBounds = (RowBounds) args[2];
        this.resultHandler = (ResultHandler) args[3];
        this.executor = (Executor) invocation.getTarget();
        List<HeavyParam> heavyParams = context.getHeavyParams();
        this.finalResult = new ArrayList<>(heavyParams.size() * singleBatchSize);
        this.heavyKeyIter = heavyParams.iterator();
    }

    public Object query() throws SQLException {
        while (heavyKeyIter.hasNext()) {
            HeavyParam currKey = heavyKeyIter.next();
            List<List<Object>> param = partitionParam(currKey.getParam());
            doQuery(currKey, param);
        }
        return finalResult;
    }

    private void doQuery(HeavyParam currKey, List<List<Object>> param) throws SQLException {
        if (!heavyKeyIter.hasNext()) {
            for (List<Object> currentParam : param) {
                updateParamMap(currKey, currentParam);
                List<Object> oneBatchResult = executor.query(ms, paramMap, rowBounds, resultHandler);
                finalResult.addAll(oneBatchResult);
            }
            return;
        } else {
            HeavyParam nextKey = heavyKeyIter.next();
            log.warn("get mutil heavy key [{}], batchSize[{}]", nextKey.shadowHeavyKeys, nextKey.getParam().size());
            List<List<Object>> nextParam = partitionParam(nextKey.getParam());
            for (List<Object> currParam : param) {
                updateParamMap(currKey, currParam);
                doQuery(nextKey, nextParam);
            }
        }
    }

    private void updateParamMap(HeavyParam currKey, List<Object> param) {
        for (String shadowKey : currKey.getShadowHeavyKeys()) {
            paramMap.put(shadowKey, param);
        }
    }
}

private HeavyParamContext findHeavyParam(Map<String, Object> parameterMap) {
    List<Map.Entry<String, Object>> heavyKeys = doFindHeavyParam(parameterMap);
    if (heavyKeys == null) {
        return BigSizeParamQueryPlugin.NO_BIG_PARAM;
    } else {
        HeavyParamContext result = new HeavyParamContext();
        List<HeavyParam> heavyParams;
        if (heavyKeys.size() == 1) {
            heavyParams = buildSingleHeavyParam(heavyKeys);
        } else {
            heavyParams = buildMultiHeavyParam(heavyKeys);
        }
        result.setHeavyParams(heavyParams);
        result.setParameter(new HashMap<>(parameterMap));
        return result;
    }
}

private List<HeavyParam> buildSingleHeavyParam(List<Map.Entry<String, Object>> heavyKeys) {
    Map.Entry<String, Object> single = heavyKeys.get(0);
    return Collections.singletonList(new HeavyParam((Collection) single.getValue(), Collections.singletonList(single.getKey())));
}

private List<List<Object>> partitionParam(Object o) {
    Collection c = (Collection) o;
    List res;
    if (c instanceof List) {
        res = (List) c.stream().distinct().collect(Collectors.toList());
    } else {
        res = new ArrayList(c);
    }
    return Lists.partition(res, singleBatchSize);
}

private List<HeavyParam> buildMultiHeavyParam(List<Map.Entry<String, Object>> heavyKeys) {
    //when heavy keys used multi time in xml, its name will be different.
    TreeMap<Collection, List<String>> params = new TreeMap<>(new Comparator<Collection>() {
        @Override
        public int compare(Collection o1, Collection o2) {
            //fixme workable but have corner case. 
            return CollectionUtils.isEqualCollection(o1, o2) == true ? 0 : o1.hashCode() - o2.hashCode();
        }
    });

    for (Map.Entry<String, Object> keyEntry : heavyKeys) {
        String key = keyEntry.getKey();
        List<String> keys = params.computeIfAbsent((Collection) keyEntry.getValue(), k -> new ArrayList<>(1));
        keys.add(key);
    }

    List<HeavyParam> hps = new ArrayList<>(params.size());
    for (Map.Entry<Collection, List<String>> heavyEntry : params.entrySet()) {
        List<String> shadowKeys = heavyEntry.getValue();
        hps.add(new HeavyParam(heavyEntry.getKey(), shadowKeys));
    }
    return hps;
}

private List<Map.Entry<String, Object>> doFindHeavyParam(Map<String, Object> parameterMap) {
    List<Map.Entry<String, Object>> result = null;
    for (Map.Entry<String, Object> p : parameterMap.entrySet()) {
        if (p != null) {
            Object value = p.getValue();
            if (value != null && value instanceof Collection) {
                int size = CollectionUtils.size(value);
                if (size > singleBatchSize) {
                    if (result == null) {
                        result = new ArrayList<>(1);
                    }
                    result.add(p);
                }
            }
        }
    }
    return result;
}


@Getter
@Setter
private static class HeavyParamContext {
    private Boolean hasHeavyParam;
    private List<HeavyParam> heavyParams;
    private Map<String, Object> parameter;

    public Boolean hasHeavyParam() {
        return heavyParams != null;
    }
}


@Data
@AllArgsConstructor
@NoArgsConstructor
private class HeavyParam {
    private Collection param;
    private List<String> shadowHeavyKeys;
}

@Override
public Object plugin(Object o) {
    return Plugin.wrap(o, this);
}

@Override
public void setProperties(Properties properties) {
}

}

myBATIS foreach 达到 1000 的限制

myBATIS foreach hitting limit of 1000

java

oracle

mybatis