目录
- 1.代码中foreach insert/update
- 2.多线程foreach insert/update
- 3.mybatis xml中foreach
- 4.mybatis-plus扩展
- not null问题
现工作中有需求要进行批量新增和修改
实现了以下几种方式
- 代码中foreach insert/update
- 多线程foreach insert/update
- mybatis xml中foreach
- mybatis-plus扩展
第一种就不说了,重复的IO连接与断开效率极低,性能很差,不考虑
第二种使用多线程进行批量插入/修改,时间会大大降低,但还会有频繁建立断开IO,性能不好
第三种其实就是在拼sql,但是不同业务要拼不同的sql,复用性很差
第四种本质也是拼sql,但是通过简单的配置就可以达到不同业务的复用
1.代码中foreach insert/update
for(int i=0;i<insertList.size();i++){ | |
offerMapper.insert(offerDO); | |
} |
更新同理
2.多线程foreach insert/update
工作中也使用过多线程批量更新,新增同理
//定义线程池 | |
private static final Long KEEP_ALIVE_TIME = 60L; | |
private static final int APS = Runtime.getRuntime().availableProcessors(); | |
private static final ThreadPoolExecutor THREAD_POOL_EXECUTOR = new ThreadPoolExecutor( | |
APS * 2, | |
APS * 4, | |
KEEP_ALIVE_TIME, | |
TimeUnit.SECONDS, | |
new LinkedBlockingDeque<>(256), | |
new ThreadFactoryBuilder().setNameFormat("分拣出库-pool-%d").build(), | |
new ThreadPoolExecutor.CallerRunsPolicy() | |
); | |
//使用 | |
try { | |
taskExecute(list, yearList); | |
} catch (Exception e) { | |
log.error("分拣出库更新失败:{}", e); | |
} | |
// 处理单个任务数据(year是分库分表用的) | |
private void taskExecute(List<SortingOutboundProductDetailDO> list, List<Integer> yearList) throws Exception { | |
if (CollectionUtils.isEmpty(list)) { | |
return; | |
} | |
final CountDownLatch latch = new CountDownLatch(list.size()); | |
for (SortingOutboundProductDetailDO data : list) { | |
THREAD_POOL_EXECUTOR.submit(() -> { | |
try { | |
//更新从表 | |
sortingOutboundProductDetailMapper.update(null, | |
new LambdaUpdateWrapper<SortingOutboundProductDetailDO>() | |
.eq(SortingOutboundProductDetailDO::getId, data.getId()) | |
.in(SortingOutboundProductDetailDO::getYear, yearList) | |
.set(SortingOutboundProductDetailDO::getOutboundNumber, data.getOutboundNumber()) | |
); | |
} finally { | |
if (latch != null) { | |
latch.countDown(); | |
} | |
} | |
}); | |
} | |
latch.await(); | |
} |
3.mybatis xml中foreach
批量新增
//不用关注里面的业务代码 | |
private int insert(OfferSaveRequest request){ | |
List<OfferDO> insertOffer = request.getOfferList().stream().map(obj -> { | |
OfferDO offerDO = new OfferDO(); | |
offerDO.setId(IdWorker.getId()); | |
offerDO.setFirstSubjectId(request.getFirstSubjectId()); | |
offerDO.setWarehouseNum(request.getWarehouseNum()); | |
offerDO.setExpressCompany(obj.getExpressCompany()); | |
offerDO.setExpressCompanyName(obj.getExpressCompanyName()); | |
offerDO.setArea(obj.getArea()); | |
offerDO.setExpensesItemName(obj.getExpensesItemName()); | |
offerDO.setUnit(obj.getUnit()); | |
offerDO.setFees(obj.getFees()); | |
offerDO.setDescription(obj.getDescription()); | |
offerDO.setTransportType(generateTransportType(obj.getExpensesItemName())); | |
offerDO.setCreateTime(new Date()); | |
offerDO.setCreateUserId(1L); | |
offerDO.setCreateUserName("管理员"); | |
return offerDO; | |
}).collect(Collectors.toList()); | |
return offerMapper.batchInsert(insertOffer); | |
} |
xml
<insert id="batchInsert" parameterType="com.model.OfferDO"> | |
INSERT INTO offer( | |
id, | |
first_subject_id, | |
warehouse_num, | |
express_company, | |
express_company_name, | |
area, | |
expenses_item_name, | |
unit, | |
fees, | |
description, | |
create_time, | |
create_user_id, | |
create_user_name | |
) | |
values | |
<foreach collection="offerList" separator="," item="offer"> | |
( | |
#{offer.id}, | |
#{offer.firstSubjectId}, | |
#{offer.warehouseNum}, | |
#{offer.expressCompany}, | |
#{offer.expressCompanyName}, | |
#{offer.area}, | |
#{offer.expensesItemName}, | |
#{offer.unit}, | |
#{offer.fees}, | |
#{offer.description}, | |
#{offer.createTime}, | |
#{offer.createUserId}, | |
#{offer.createUserName} | |
) | |
</foreach> | |
</insert> |
批量修改
//不用关注里面的业务代码 | |
List<OfferSaveRequest.Offer> updateList = request.getOfferList().stream() | |
.filter(obj -> obj.getId() != null).collect(Collectors.toList()); | |
if (updateList.size() > 0) { | |
List<OfferDO> updateOffer = updateList.stream().map(obj -> { | |
OfferDO offerDO = new OfferDO(); | |
offerDO.setId(obj.getId()); | |
offerDO.setArea(obj.getArea()); | |
offerDO.setFees(obj.getFees()); | |
offerDO.setDescription(obj.getDescription()); | |
offerDO.setUpdateTime(new Date()); | |
offerDO.setUpdateUserId(1L); | |
offerDO.setUpdateUserName("管理员"); | |
return offerDO; | |
}).collect(Collectors.toList()); | |
offerMapper.batchUpdate(updateOffer); | |
} |
xml
<update id="batchUpdate" parameterType="com.model.OfferDO"> | |
<foreach collection="offerList" item="offer" separator=";"> | |
update | |
offer | |
set | |
<if test="offer.area!=null and offer.area!=''"> | |
area=#{offer.area}, | |
</if> | |
<if test="offer.fees!=null"> | |
fees=#{offer.fees}, | |
</if> | |
<if test="offer.description!=null and offer.description!=''"> | |
description=#{offer.description}, | |
</if> | |
update_time=#{offer.updateTime}, | |
update_user_id=#{offer.updateUserId}, | |
update_user_name=#{offer.updateUserName} | |
where | |
id = #{offer.id} | |
</foreach> | |
</update> |
批量修改还需要在配置文件中配置&allowMultiQueries=true
,否则报错
spring.datasource.url=jdbc:mysql://127.0.0.1:3306/test?useSSL=false&useUnicode=true&characterEncoding=utf-8&serverTimezone=GMT%2B8&allowMultiQueries=true
4.mybatis-plus扩展
1.创建sql注入器
/** | |
* 自定义方法SQL注入器 | |
* 【注意】这个类名,可以随便命名 | |
*/ | |
public class MyInjector extends DefaultSqlInjector { | |
/** | |
* 如果只需增加方法,保留MyBatis plus自带方法, | |
* 可以先获取super.getMethodList(),再添加add | |
*/ | |
@Override | |
public List<AbstractMethod> getMethodList(Class<?> mapperClass) { | |
// 注意:此SQL注入器继承了DefaultSqlInjector(默认注入器),调用了DefaultSqlInjector的getMethodList方法,保留了mybatis-plus的自带方法 | |
List<AbstractMethod> methodList = super.getMethodList(mapperClass); | |
methodList.add(new InsertBatchMethod()); | |
methodList.add(new UpdateBatchMethod()); | |
return methodList; | |
} | |
} |
2.注入容器
@Configuration | |
@MapperScan("com.yida.mapper") | |
public class MybatisPlusPageConfig { | |
@Bean | |
public MyInjector myInjector(){ | |
return new MyInjector(); | |
} | |
} |
3.定义通用mapper
/** | |
* 公共mapper | |
* 要实现批量新增/修改 继承此类 | |
* | |
* @param <T> | |
*/ | |
public interface CommonMapper<T> extends BaseMapper<T> { | |
/** | |
* 自定义批量插入 | |
* 如果要自动填充,@Param(xx) xx参数名必须是 list/collection/array 3个的其中之一 | |
*/ | |
int insertBatch(; List<T> list) | |
/** | |
* 自定义批量更新,条件为主键 | |
* 如果要自动填充,@Param(xx) xx参数名必须是 list/collection/array 3个的其中之一 | |
*/ | |
int updateBatch(; List<T> list) | |
} |
4.新增/修改
/** | |
* 批量新增 | |
*/ | |
public class InsertBatchMethod extends AbstractMethod { | |
/** | |
* insert into user(id, name, age) values (1, "a", 17), (2, "b", 18); | |
<script> | |
insert into user(id, name, age) values | |
<foreach collection="list" item="item" index="index" open="(" separator="),(" close=")"> | |
#{item.id}, #{item.name}, #{item.age} | |
</foreach> | |
</script> | |
*/ | |
public MappedStatement injectMappedStatement(Class<?> mapperClass, Class<?> modelClass, TableInfo tableInfo) { | |
final String sql = "<script>insert into %s %s values %s</script>"; | |
final String fieldSql = prepareFieldSql(tableInfo); | |
final String valueSql = prepareValuesSql(tableInfo); | |
final String sqlResult = String.format(sql, tableInfo.getTableName(), fieldSql, valueSql); | |
log.debug("sqlResult----->{}", sqlResult); | |
SqlSource sqlSource = languageDriver.createSqlSource(configuration, sqlResult, modelClass); | |
// 第三个参数必须和RootMapper的自定义方法名一致 | |
return this.addInsertMappedStatement(mapperClass, modelClass, "insertBatch", sqlSource, new NoKeyGenerator(), null, null); | |
} | |
private String prepareFieldSql(TableInfo tableInfo) { | |
StringBuilder fieldSql = new StringBuilder(); | |
fieldSql.append(tableInfo.getKeyColumn()).append(","); | |
tableInfo.getFieldList().forEach(x -> { | |
//新增时修改字段不填充 | |
if (!("update_time".equals(x.getColumn())) | |
&&!("update_user_id".equals(x.getColumn())) | |
&&!("update_user_name".equals(x.getColumn()))){ | |
fieldSql.append(x.getColumn()).append(","); | |
} | |
}); | |
fieldSql.delete(fieldSql.length() - 1, fieldSql.length()); | |
fieldSql.insert(0, "("); | |
fieldSql.append(")"); | |
return fieldSql.toString(); | |
} | |
private String prepareValuesSql(TableInfo tableInfo) { | |
final StringBuilder valueSql = new StringBuilder(); | |
valueSql.append("<foreach collection=\"list\" item=\"item\" index=\"index\" open=\"(\" separator=\"),(\" close=\")\">"); | |
valueSql.append("#{item.").append(tableInfo.getKeyProperty()).append("},"); | |
tableInfo.getFieldList().forEach(x -> { | |
if (!("updateTime".equals(x.getProperty())) | |
&&!("updateUserId".equals(x.getProperty())) | |
&&!("updateUserName".equals(x.getProperty()))){ | |
valueSql.append("#{item.").append(x.getProperty()).append("},"); | |
} | |
}); | |
valueSql.delete(valueSql.length() - 1, valueSql.length()); | |
valueSql.append("</foreach>"); | |
return valueSql.toString(); | |
} | |
} | |
/** | |
* 批量更新方法实现,条件为主键,选择性更新 | |
*/ | |
public class UpdateBatchMethod extends AbstractMethod { | |
/** | |
* update user set name = "a", age = 17 where id = 1; | |
* update user set name = "b", age = 18 where id = 2; | |
<script> | |
<foreach collection="list" item="item" separator=";"> | |
update user | |
<set> | |
<if test="item.name != null and item.name != ''"> | |
name = #{item.name,jdbcType=VARCHAR}, | |
</if> | |
<if test="item.age != null"> | |
age = #{item.age,jdbcType=INTEGER}, | |
</if> | |
</set> | |
where id = #{item.id,jdbcType=INTEGER} | |
</foreach> | |
</script> | |
*/ | |
public MappedStatement injectMappedStatement(Class<?> mapperClass, Class<?> modelClass, TableInfo tableInfo) { | |
String sql = "<script>\n<foreach collection=\"list\" item=\"item\" separator=\";\">\nupdate %s %s where %s=#{%s} %s\n</foreach>\n</script>"; | |
String additional = tableInfo.isWithVersion() ? tableInfo.getVersionFieldInfo().getVersionOli("item", "item.") : "" + tableInfo.getLogicDeleteSql(true, true); | |
String setSql = sqlSet(tableInfo.isWithLogicDelete(), false, tableInfo, false, "item", "item."); | |
String sqlResult = String.format(sql, tableInfo.getTableName(), setSql, tableInfo.getKeyColumn(), "item." + tableInfo.getKeyProperty(), additional); | |
log.debug("sqlResult----->{}", sqlResult); | |
SqlSource sqlSource = languageDriver.createSqlSource(configuration, sqlResult, modelClass); | |
// 第三个参数必须和RootMapper的自定义方法名一致 | |
return this.addUpdateMappedStatement(mapperClass, modelClass, "updateBatch", sqlSource); | |
} | |
} |
5.使用,将原有的继承BaseMapper的方法,改写为继承CommonMapper,后续批量操作,直接使用新增的两个方法进行处理即可。
public interface OfferMapper extends CommonMapper<OfferDO> { | |
} |
新增:offerMapper.insertBatch(insertOffer)
更新:offerMapper.updateBatch(updateOffer)
not null问题
在实际使用中发现一个问题,这个批量插入是在项目启动后就进行拼接好的sql,然后调用的时候,进行值得替换,例
<script> | |
INSERT INTO express (id,express_name,express_code,state,create_time,create_user_id,create_user_name) VALUES <foreach collection="list" item="et" separator=","> | |
(#{id},#{expressName},#{expressCode},#{state},#{createTime},#{createUserId},#{createUserName}) | |
</foreach> | |
</script> |
发现是全量新增,这样也就产生了一个问题,当只想新增一部分数据,剩下的一部分数据更新进去时,有时
会有问题,这取决于你数据库中字段设置是可以为null还是不可以为null。
当数据库中字段设置为not null,而新增的时候传一个null,就会触发数据库的not null校验,报错
然后就查资料,改代码,测试,然后官网上是这样说的
参考:https://blog.csdn.net/weixin_45505313/article/details/121574166
Inserting NULL into a column that has been declared NOT NULL. For multiple-row INSERT statements or | |
INSERT INTO ... SELECT statements, the column is set to the implicit default value for the column | |
data type. This is 0 for numeric types, the empty string ('') for string types, and the “zero” value | |
for date and time types. INSERT INTO ... SELECT statements are handled the same way as multiple-row | |
inserts because the server does not examine the result set from the SELECT to see whether it returns | |
a single row. (For a single-row INSERT, no warning occurs when NULL is inserted into a NOT NULL column. | |
Instead, the statement fails with an error.) | |
在声明为NOT NULL的列中插入NULL。对于多行INSERT语句或 | |
插入…SELECT语句时,该列被设置为该列的隐式默认值 | |
数据类型。数值类型为0,字符串类型为空字符串("),值为" 0 " | |
用于日期和时间类型。插入…SELECT语句的处理方式与多行语句相同 | |
插入,因为服务器不检查SELECT的结果集,看它是否返回 | |
单行。(对于单行INSERT,当NULL插入到NOT NULL列时,不会出现警告。 | |
相反,语句失败并报错。) |
也就是说mysql允许批量插入时,向not null字段插入null值,mysql会给其赋一个隐藏值
但是在我实测下发现并不行,然后又开始查资料,被我发现了这个
那么我就查了一下我的数据库模式
select @@sql_mode; | |
结果:ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION |
可以看出我的数据库模式为严格模式,怪不得官网说可以插入null,而我的代码一直报错,排坑之旅任重道远
解决方案:
1.关闭数据库的严格模式(公司的数据库没有权限,这个直接pass掉)
2.手动拼批量插入的sql,拼成如下样子,然后一次请求执行,这样只进行了一次数据库连接,也可以实现批量插入的效果,但是不知道这种多个INSERT语句与单个INSERT和多个VALUES的性能怎么样
INSERT INTO `oss` VALUES (1, -1, '', '测试用文件.docx', '', '', 0, '', '2022-12-08 16:21:33', 1, '系统管理员', NULL, -1, ''); | |
INSERT INTO `oss` VALUES (2, -1, '', '测试用文件.docx', '', '', 0, '', '2022-12-08 16:32:32', 1, '系统管理员', NULL, -1, ''); | |
INSERT INTO `oss` VALUES (3, -1, '', '测试用文件.docx', '', '', 0, '', '2022-12-08 16:33:17', 1, '系统管理员', NULL, -1, ''); | |
INSERT INTO `oss` VALUES (4, -1, '', '测试用文件.docx', '', '', 0, '', '2022-12-08 16:44:30', 1, '系统管理员', NULL, -1, ''); | |
INSERT INTO `oss` VALUES (5, -1, '', '测试用文件.docx', '', '', 0, '', '2022-12-08 16:45:28', 1, '系统管理员', NULL, -1, ''); | |
INSERT INTO `oss` VALUES (6, -1, '', '合同导入测试.doc','', '', 0, '', '2022-12-08 16:47:03', 1, '系统管理员', NULL, -1, ''); | |
INSERT INTO `oss` VALUES (7, -1, '', '合同导入测试.doc','', '', 0, '', '2022-12-08 16:48:03', 1, '系统管理员', NULL, -1, ''); | |
INSERT INTO `oss` VALUES (8, -1, '', '测试用文件.docx', '', '', 1, '', '2022-12-08 16:49:35', 1, '系统管理员', NULL, -1, ''); | |
INSERT INTO `oss` VALUES (9, -1, '', '新建文本文档.doc','', '', 0, '', '2022-12-08 17:12:36', 1, '系统管理员', NULL, -1, ''); |