SQL优化

src/dashoo.cn/backend/api/business/tmpzcgf/tmpzcgfSession.go

@@ -121,14 +121,11 @@ FROM
 
 func (s *TmpzcgtSession) InsertSupplierCert() error {
 	s.DBE.Query("SET unique_checks=0;")
-	s.DBE.Query("SET autocommit=0;")
-	sql := "INSERT ignore INTO OilSupplierCert (SupplierId,OutsideFlog, `Status` , Step, WorkerTotal, AccessCardNo,SupplierTypeName,InFlag,EffectStartTime,EffectEndTime,ApplyTime,Remark,CreateBy,GfID) " +
+	sql := "INSERT INTO OilSupplierCert (SupplierId,OutsideFlog, `Status` , Step, WorkerTotal, AccessCardNo,SupplierTypeName,InFlag,EffectStartTime,EffectEndTime,ApplyTime,Remark,CreateBy,GfID) " +
 		"SELECT -1 As SupplierId,企业名称 AS OutsideFlog,'8' As `Status`, 3 as Step, 1 as WorkerTotal, 准入证号 As AccessCardNo,准入类别 AS SupplierTypeName,准入标识 AS InFlag, STR_TO_DATE(IF(有效期起='', NULL, 有效期起),'%Y年%m月%d日') AS EffectStartTime, STR_TO_DATE(IF(有效期止='', NULL, 有效期止),'%Y年%m月%d日') AS EffectEndTime,(SELECT STR_TO_DATE(IF(MAX(年审日期)='', NULL, MAX(年审日期)),'%Y-%m-%d') from T_年审 b WHERE b.企业名称 = a.企业名称 AND b.状态标识='办理完毕') AS ApplyTime,备注 AS Remark,'导入' As CreateBy,ID " +
 		"FROM T_资格预审表 a ;" //WHERE 准入标识 != '取消'
 	// s.DBE.Query("SET @@SESSION.sql_mode='ALLOW_INVALID_DATES'")
 	_, err := s.DBE.Query(sql)
-	s.DBE.Query("commit;")
-	s.DBE.Query("SET autocommit=1;")
 	s.DBE.Query("SET unique_checks=1;")
 	return err
 }
@@ -155,7 +152,7 @@ func (s *TmpzcgtSession) UpdateSupplierTypeCodeAndInFlag() error {
 	if err != nil {
 		return err
 	}
-	sql31 := "UPDATE `OilSupplierCert` SET SupplierTypeCode = '03' WHERE SupplierTypeName='技术服务类'"
+	sql31 := "UPDATE `OilSupplierCert` SET SupplierTypeCode = '03', SupplierTypeName='服务类' WHERE SupplierTypeName='技术服务类'"
 	_, err = s.DBE.Query(sql31)
 	if err != nil {
 		return err
@@ -177,15 +174,13 @@ func (s *TmpzcgtSession) UpdateSupplierTypeCodeAndInFlag() error {
 }
 
 func (s *TmpzcgtSession) InsterAnnual() error {
-	sqlAlterRemark := "ALTER TABLE `OilSupplierCertAppend`\n\tCHANGE COLUMN `Remark` `Remark` VARCHAR(5000) NULL DEFAULT NULL COMMENT '备注' COLLATE 'utf8_general_ci' AFTER `Status`;"
-	_, err := s.DBE.Query(sqlAlterRemark)
 
 	sql := "INSERT INTO OilAnnualAudit (Status,SupplierTypeName, SupplierName, CreateOn, CreateBy, RecUnitId, ApplyTime, AccessCardNo, OldId) " +
 		"SELECT '11' as Status ,(CASE 准入类别 WHEN '物资类' THEN '01' WHEN '基建类' THEN '02' WHEN '服务类' THEN '03' WHEN '技术服务类' THEN '03' END ) SupplierTypeName, " +
 		"企业名称 AS SupplierName, STR_TO_DATE(IF(申请日期='', NULL, 申请日期),'%Y-%m-%d') AS CreateOn, 录入员 AS CreateBy, 推荐单位编码 AS RecUnitId, STR_TO_DATE(IF(年审日期='', NULL, 年审日期),'%Y-%m-%d') AS ApplyTime, 准入证号 AS AccessCardNo, id AS oldId " +
 		"FROM T_年审"
 
-	_, err = s.DBE.Query(sql)
+	_, err := s.DBE.Query(sql)
 	return err
 }
 
@@ -236,7 +231,7 @@ func (s *TmpzcgtSession) InsterBadRecord() error {
 		") " +
 		"SELECT " +
 		"gfid AS Bak1 , " +
-		"日期 AS CreateOn, " +
+		"STR_TO_DATE(IF(日期='', NULL, 日期),'%Y-%m-%d') AS CreateOn, " +
 		"内容 AS Record, " +
 		"'' AS CompanyName, " +
 		"'' AS CompanyCode, " +
@@ -285,7 +280,6 @@ func (s *TmpzcgtSession) InsertCertSub() error {
 		}
 		n := math.Ceil(float64(total) / 5000)
 
-		s.DBE.Query("set session bulk_insert_buffer_size = 1024 * 1024 * 120;")
 		s.DBE.Query("ALTER TABLE `OilSupplierCertSub` DISABLE KEYS;")
 		s.DBE.Query("SET unique_checks=0;")
 		s.DBE.Query("SET autocommit=0;")
@@ -335,6 +329,8 @@ func (s *TmpzcgtSession) UpdateCertSub() error {
 	s.GetEntitysByWhere("OilSupplierCert", "", &certList)
 
 	var err error
+
+	s.DBE.Query("ALTER TABLE `OilSupplierCertSub` ADD INDEX index_oldid ( `OldId` );")
 	for _, cert := range certList {
 		sql := " UPDATE OilSupplierCertSub b SET b.SupplierId = '" + strconv.Itoa(cert.SupplierId) + "'," +
 			" b.SupplierCertId='"+ strconv.Itoa(cert.Id) + "',"+
@@ -345,6 +341,7 @@ func (s *TmpzcgtSession) UpdateCertSub() error {
 
 		_, err = s.DBE.Query(sql)
 	}
+	s.DBE.Query("alter table OilSupplierCertSub drop index index_oldid ;")
 	return err
 }
 
@@ -380,7 +377,14 @@ func (s *TmpzcgtSession) InsertOilSupplierCertAppend() error {
 	//	"from tmp_zengxiang "+
 	//	"group by id,申请日期,企业名称,录入员,推荐单位编码,申请日期,准入类别"
 	// countSql := "SELECT count(*) from (select a.申请日期 from tmp_zengxiang a WHERE 企业名称 != '' GROUP BY a.`申请日期`, a.`企业名称`, a.`准入类别`) b"
-	countSql := "select count(distinct a.`申请日期`, a.`企业名称`, a.`准入类别`) from T_增项 a WHERE 企业名称 != ''"
+	sqlAlterRemark := "ALTER TABLE `OilSupplierCertAppend` CHANGE COLUMN `Remark` `Remark` MEDIUMTEXT NULL COMMENT '备注' COLLATE 'utf8_general_ci' AFTER `Status`;"
+	_, err := s.DBE.Query(sqlAlterRemark)
+
+	// s.DBE.Query("ALTER TABLE `T_增项` ADD INDEX index_import ( `申请日期`, `企业名称`, `准入类别` );")
+
+	s.DBE.Query("SET GLOBAL group_concat_max_len=1024000;")
+
+	/*countSql := "select count(distinct a.`申请日期`, a.`企业名称`, a.`准入类别`) from T_增项 a WHERE 企业名称 != ''"
 	countStr, err := s.DBE.Query(countSql)
 	var total int64
 	if len(countStr) > 0 {
@@ -390,25 +394,26 @@ func (s *TmpzcgtSession) InsertOilSupplierCertAppend() error {
 			break
 		}
 		n := math.Ceil(float64(total) / 2000)
-
-		s.DBE.Query("SET unique_checks=0;")
-		s.DBE.Query("SET autocommit=0;")
 		for i:=0; i <= int(n); i++ {
 			start := strconv.Itoa(i * 2000)
 			end := strconv.Itoa(2000)
-			sql := "INSERT INTO OilSupplierCertAppend " +
-				"(Status,CreateOn,SupplierName,AppendType,Remark,CreateBy,ApplyDate) " +
-				"SELECT '11' as Status, a.`申请日期` AS CreateOn, a.`企业名称` AS SupplierName, (CASE 准入类别 WHEN '物资类' THEN '01' WHEN '基建类' THEN '02' WHEN '服务类' THEN '03' WHEN '技术服务类' THEN '03' END) AppendType, " +
-				"group_concat(a.id) as Remark, '导入' AS CreateBy, a.`申请日期` AS ApplyDate " +
-				"FROM T_增项 a WHERE 企业名称 != '' GROUP BY a.`申请日期`, a.`企业名称`, a.`准入类别` ORDER BY a.`申请日期`, a.`准入类别`, a.`准入类别` " +
-				"limit " + start + "," +  end
-			_, err = s.DBE.Query(sql)
-			s.DBE.Query("commit;")
+
 		}
+	}*/
 
-		s.DBE.Query("SET unique_checks=1;")
-		s.DBE.Query("SET autocommit=1;")
-	}
+	s.DBE.Query("SET unique_checks=0;")
+
+	sql := "INSERT INTO OilSupplierCertAppend " +
+		"(OldId,Status,CreateOn,SupplierName,AppendType,Remark,CreateBy,ApplyDate) " +
+		"SELECT Max(id) as OldId, '11' as Status,STR_TO_DATE(IF( a.`申请日期`='', NULL,  a.`申请日期`),'%Y-%m-%d') AS CreateOn, a.`企业名称` AS SupplierName, (CASE 准入类别 WHEN '物资类' THEN '01' WHEN '基建类' THEN '02' WHEN '服务类' THEN '03' WHEN '技术服务类' THEN '03' END) AppendType, " +
+		"group_concat(a.`物资编码`) as Remark, '导入' AS CreateBy, STR_TO_DATE(IF( a.`申请日期`='', NULL,  a.`申请日期`),'%Y-%m-%d') AS ApplyDate " +
+		"FROM T_增项 a WHERE 企业名称 != '' GROUP BY a.`申请日期`, a.`企业名称`, a.`准入类别` "
+	// " ORDER BY a.`申请日期`, a.`企业名称`, a.`准入类别` "
+	_, err = s.DBE.Query(sql)
+
+	s.DBE.Query("SET unique_checks=1;")
+
+	// s.DBE.Query("ALTER TABLE `T_增项` drop INDEX index_import;")
 
 	return err
 }
@@ -507,7 +512,6 @@ func (s *TmpzcgtSession) UpdateOilSupplierCertAppendSub() error {
 	var err error
 	s.DBE.Query("SET unique_checks=0;")
 	s.DBE.Query("SET autocommit=0;")
-	s.DBE.Query("ALTER TABLE `OilSupplierCertSub` ADD INDEX index_oldid ( `OldId` );")
 	for _, certAppend := range certAppendList {
 		appendCodeList := strings.Split(certAppend.Remark, ",")
 		for _, appendCode := range appendCodeList {
@@ -518,7 +522,6 @@ func (s *TmpzcgtSession) UpdateOilSupplierCertAppendSub() error {
 			s.DBE.Query("commit;")
 		}
 	}
-	s.DBE.Query("alter table OilSupplierCertSub drop index index_oldid ;")
 	s.DBE.Query("SET unique_checks=1;")
 	s.DBE.Query("SET autocommit=1;")
 	return err

src/dashoo.cn/backend/api/controllers/tmpzcgf/tmpzcgf.go

@@ -845,6 +845,10 @@ func (this *TmpzcgfController) ClearAll() {
 
 	sql = "truncate table OilSupplierSceneFile"
 	svc.DBE.Query(sql)
+
+	// 设置环境变量
+	svc.DBE.Query("set session bulk_insert_buffer_size = 120 * 1024 * 1024;")
+
 	fmt.Println("==========truncate table结束=========")
 
 }
@@ -868,7 +872,7 @@ func (this *TmpzcgfController) InsertAll() {
 		this.UpdateOilSupplierCertAppend()
 		this.UpdateOilSupplierCertAppendSub() // 更新增项的准入范围
 
-		// this.CapacityExpansion() // 编码扩充
+		// this.CapacityExpansion() // 编码扩充 --弃用
 		// 历史数据导入-由于运行到此处中断，上面的代码临时注释
 		this.InsterAnnualAudit()
 		this.UpdateAnnualAudit()