优化执行导入SQL执行效率

src/dashoo.cn/backend/api/business/tmpzcgf/tmpzcgfSession.go

@@ -1,6 +1,7 @@
 package tmpzcgf
 
 import (
+	"dashoo.cn/backend/api/business/oilsupplier/suppliercert"
 	"dashoo.cn/backend/api/business/oilsupplier/suppliercertappend"
 	. "dashoo.cn/backend/api/mydb"
 	"github.com/go-xorm/xorm"
@@ -119,11 +120,16 @@ FROM
 }
 
 func (s *TmpzcgtSession) InsertSupplierCert() error {
-	sql := "INSERT INTO OilSupplierCert (SupplierId,OutsideFlog, `Status` , Step, WorkerTotal, AccessCardNo,SupplierTypeName,InFlag,EffectStartTime,EffectEndTime,ApplyTime,Remark,CreateBy,GfID) " +
+	s.DBE.Query("SET unique_checks=0;")
+	s.DBE.Query("SET autocommit=0;")
+	sql := "INSERT ignore INTO OilSupplierCert (SupplierId,OutsideFlog, `Status` , Step, WorkerTotal, AccessCardNo,SupplierTypeName,InFlag,EffectStartTime,EffectEndTime,ApplyTime,Remark,CreateBy,GfID) " +
 		"SELECT -1 As SupplierId,企业名称 AS OutsideFlog,'8' As `Status`, 3 as Step, 1 as WorkerTotal, 准入证号 As AccessCardNo,准入类别 AS SupplierTypeName,准入标识 AS InFlag, STR_TO_DATE(IF(有效期起='', NULL, 有效期起),'%Y年%m月%d日') AS EffectStartTime, STR_TO_DATE(IF(有效期止='', NULL, 有效期止),'%Y年%m月%d日') AS EffectEndTime,(SELECT STR_TO_DATE(IF(MAX(年审日期)='', NULL, MAX(年审日期)),'%Y-%m-%d') from T_年审 b WHERE b.企业名称 = a.企业名称 AND b.状态标识='办理完毕') AS ApplyTime,备注 AS Remark,'导入' As CreateBy,ID " +
 		"FROM T_资格预审表 a ;" //WHERE 准入标识 != '取消'
-	s.DBE.Query("SET @@SESSION.sql_mode='ALLOW_INVALID_DATES'")
+	// s.DBE.Query("SET @@SESSION.sql_mode='ALLOW_INVALID_DATES'")
 	_, err := s.DBE.Query(sql)
+	s.DBE.Query("commit;")
+	s.DBE.Query("SET autocommit=1;")
+	s.DBE.Query("SET unique_checks=1;")
 	return err
 }
 
@@ -171,12 +177,15 @@ func (s *TmpzcgtSession) UpdateSupplierTypeCodeAndInFlag() error {
 }
 
 func (s *TmpzcgtSession) InsterAnnual() error {
+	sqlAlterRemark := "ALTER TABLE `OilSupplierCertAppend`\n\tCHANGE COLUMN `Remark` `Remark` VARCHAR(5000) NULL DEFAULT NULL COMMENT '备注' COLLATE 'utf8_general_ci' AFTER `Status`;"
+	_, err := s.DBE.Query(sqlAlterRemark)
+
 	sql := "INSERT INTO OilAnnualAudit (Status,SupplierTypeName, SupplierName, CreateOn, CreateBy, RecUnitId, ApplyTime, AccessCardNo, OldId) " +
 		"SELECT '11' as Status ,(CASE 准入类别 WHEN '物资类' THEN '01' WHEN '基建类' THEN '02' WHEN '服务类' THEN '03' WHEN '技术服务类' THEN '03' END ) SupplierTypeName, " +
-		"企业名称 AS SupplierName, 申请日期 AS CreateOn, 录入员 AS CreateBy, 推荐单位编码 AS RecUnitId, 年审日期 AS ApplyTime, 准入证号 AS AccessCardNo, id AS oldId " +
+		"企业名称 AS SupplierName, STR_TO_DATE(IF(申请日期='', NULL, 申请日期),'%Y-%m-%d') AS CreateOn, 录入员 AS CreateBy, 推荐单位编码 AS RecUnitId, STR_TO_DATE(IF(年审日期='', NULL, 年审日期),'%Y-%m-%d') AS ApplyTime, 准入证号 AS AccessCardNo, id AS oldId " +
 		"FROM T_年审"
 
-	_, err := s.DBE.Query(sql)
+	_, err = s.DBE.Query(sql)
 	return err
 }
 
@@ -192,17 +201,17 @@ func (s *TmpzcgtSession) InsertOilPaymentInfo() error {
 		"PayMode," +
 		"CreateBy" +
 		")" +
-		"SELECT ID_GF, 企业名称 AS SupplierName ,交费日期 AS PayDate, 新准入 AS Amount, '2' AS IsPay,'1' AS PayType , 备注 AS Remark, 交款方式 AS PayMode,收款人 AS CreateBy FROM T_交费记录 WHERE 新准入 IS NOT NULL" +
+		"SELECT ID_GF, 企业名称 AS SupplierName ,交费日期 AS PayDate, IF(新准入='', 0, 新准入)  AS Amount, '2' AS IsPay,'1' AS PayType , 备注 AS Remark, 交款方式 AS PayMode,收款人 AS CreateBy FROM T_交费记录 WHERE 新准入 IS NOT NULL and 新准入<>''" +
 		" UNION" +
-		" SELECT ID_GF, 企业名称 AS SupplierName ,交费日期 AS PayDate, 换证 AS Amount , '2' AS IsPay, '4' AS PayType , 备注 AS Remark, 交款方式 AS PayMode,收款人 AS CreateBy FROM T_交费记录 WHERE 换证 IS NOT NULL" +
+		" SELECT ID_GF, 企业名称 AS SupplierName ,交费日期 AS PayDate, IF(换证='', 0, 换证) AS Amount , '2' AS IsPay, '4' AS PayType , 备注 AS Remark, 交款方式 AS PayMode,收款人 AS CreateBy FROM T_交费记录 WHERE 换证 IS NOT NULL and 换证<>''" +
 		" UNION" +
-		" SELECT ID_GF, 企业名称 AS SupplierName ,交费日期 AS PayDate, 增项 AS Amount , '2' AS IsPay, '3' AS PayType , 备注 AS Remark, 交款方式 AS PayMode,收款人 AS CreateBy FROM T_交费记录 WHERE 增项 IS NOT NULL" +
+		" SELECT ID_GF, 企业名称 AS SupplierName ,交费日期 AS PayDate, IF(增项='', 0, 增项) AS Amount , '2' AS IsPay, '3' AS PayType , 备注 AS Remark, 交款方式 AS PayMode,收款人 AS CreateBy FROM T_交费记录 WHERE 增项 IS NOT NULL and 增项<>''" +
 		" UNION" +
-		" SELECT ID_GF, 企业名称 AS SupplierName,交费日期 AS PayDate, 年审 AS Amount , '2' AS IsPay, '2' AS PayType , 备注 AS Remark, 交款方式 AS PayMode,收款人 AS CreateBy FROM T_交费记录 WHERE 年审 IS NOT NULL" +
+		" SELECT ID_GF, 企业名称 AS SupplierName,交费日期 AS PayDate, IF(年审='', 0, 年审) AS Amount , '2' AS IsPay, '2' AS PayType , 备注 AS Remark, 交款方式 AS PayMode,收款人 AS CreateBy FROM T_交费记录 WHERE 年审 IS NOT NULL and 年审<>''" +
 		" UNION" +
-		" SELECT ID_GF, 企业名称 AS SupplierName,交费日期 AS PayDate, 罚款 AS Amount , '2' AS IsPay, '5' AS PayType , 备注 AS Remark, 交款方式 AS PayMode,收款人 AS CreateBy FROM T_交费记录 WHERE 罚款 IS NOT NULL" +
+		" SELECT ID_GF, 企业名称 AS SupplierName,交费日期 AS PayDate, IF(罚款='', 0, 罚款) AS Amount , '2' AS IsPay, '5' AS PayType , 备注 AS Remark, 交款方式 AS PayMode,收款人 AS CreateBy FROM T_交费记录 WHERE 罚款 IS NOT NULL and 罚款<>''" +
 		" UNION" +
-		" SELECT ID_GF, 企业名称 AS SupplierName,交费日期 AS PayDate, 其它 AS Amount , '2' AS IsPay, '6' AS PayType , 备注 AS Remark, 交款方式 AS PayMode,收款人 AS CreateBy FROM T_交费记录 WHERE 其它 IS NOT NULL"
+		" SELECT ID_GF, 企业名称 AS SupplierName,交费日期 AS PayDate, IF(其它='', 0, 其它) AS Amount , '2' AS IsPay, '6' AS PayType , 备注 AS Remark, 交款方式 AS PayMode,收款人 AS CreateBy FROM T_交费记录 WHERE 其它 IS NOT NULL and 其它<>''"
 
 	_, err := s.DBE.Query(sql)
 	return err
@@ -274,11 +283,17 @@ func (s *TmpzcgtSession) InsertCertSub() error {
 			total, err = strconv.ParseInt(string(value), 10, 64)
 			break
 		}
-		n := math.Ceil(float64(total) / 2000)
+		n := math.Ceil(float64(total) / 5000)
+
+		s.DBE.Query("set session bulk_insert_buffer_size = 1024 * 1024 * 120;")
+		s.DBE.Query("ALTER TABLE `OilSupplierCertSub` DISABLE KEYS;")
+		s.DBE.Query("SET unique_checks=0;")
+		s.DBE.Query("SET autocommit=0;")
+
 		for i:=0; i <= int(n); i++ {
-			start := strconv.Itoa(i * 2000)
-			end := strconv.Itoa(2000)
-			sql := "INSERT `OilSupplierCertSub` (" +
+			start := strconv.Itoa(i * 5000)
+			end := strconv.Itoa(5000)
+			sql := "INSERT INTO `OilSupplierCertSub` (" +
 				"OldId," +
 				"AppendStatus," +
 				"`Type`," +
@@ -302,7 +317,11 @@ func (s *TmpzcgtSession) InsertCertSub() error {
 				ORDER BY t.auto_id ` +
 				" limit " + start + "," +  end
 			_, err = s.DBE.Query(sql)
+			s.DBE.Query("COMMIT;")
 		}
+		s.DBE.Query("SET autocommit=1;")
+		s.DBE.Query("SET unique_checks=1;")
+		s.DBE.Query("ALTER TABLE `OilSupplierCertSub` ENABLE KEYS;")
 	}
 	return err
 }
@@ -311,12 +330,21 @@ func (s *TmpzcgtSession) UpdateCertSub() error {
 	/*sql := "UPDATE OilSupplierCertSub b SET b.SupplierId = IFNULL((SELECT a.SupplierId FROM OilSupplierCert a WHERE a.GfId =b.OldId AND a.SupplierTypeCode = b.SupplierTypeCode AND a.SupplierId IS NOT NULL limit 1 ), 0)," +
 		"b.SupplierCertId = IFNULL((SELECT a.Id FROM OilSupplierCert a WHERE a.GfId =b.OldId AND a.SupplierTypeCode = b.SupplierTypeCode limit 1), 0) " +
 		" where b.Type = '1' and b.CreateBy='导入1'"*/
+	var certList [] suppliercert.OilSupplierCert
+	//svc.GetEntitysByWhere(OilSupplierCertAppendSubName, where, &list)
+	s.GetEntitysByWhere("OilSupplierCert", "", &certList)
 
-	sql := "UPDATE OilSupplierCertSub b left join OilSupplierCert a on (a.GfId =b.OldId AND a.SupplierTypeCode = b.SupplierTypeCode)  SET b.SupplierId = IFNULL(a.SupplierId, 0)," +
-		"b.SupplierCertId = IFNULL(a.Id, 0), b.SupplierTypeCode = a.SupplierTypeCode " +
-		" where b.Type = '1' and b.CreateBy='导入1' AND a.SupplierId IS NOT NULL"
+	var err error
+	for _, cert := range certList {
+		sql := " UPDATE OilSupplierCertSub b SET b.SupplierId = '" + strconv.Itoa(cert.SupplierId) + "'," +
+			" b.SupplierCertId='"+ strconv.Itoa(cert.Id) + "',"+
+			" b.SupplierTypeCode='" + cert.SupplierTypeCode + "'" +
+			" where b.OldId = '" + strconv.Itoa(cert.GfId) + "'"
 
-	_, err := s.DBE.Query(sql)
+		   // "and b.Type = '1' and b.CreateBy='导入1'"
+
+		_, err = s.DBE.Query(sql)
+	}
 	return err
 }
 
@@ -362,6 +390,9 @@ func (s *TmpzcgtSession) InsertOilSupplierCertAppend() error {
 			break
 		}
 		n := math.Ceil(float64(total) / 2000)
+
+		s.DBE.Query("SET unique_checks=0;")
+		s.DBE.Query("SET autocommit=0;")
 		for i:=0; i <= int(n); i++ {
 			start := strconv.Itoa(i * 2000)
 			end := strconv.Itoa(2000)
@@ -372,13 +403,19 @@ func (s *TmpzcgtSession) InsertOilSupplierCertAppend() error {
 				"FROM T_增项 a WHERE 企业名称 != '' GROUP BY a.`申请日期`, a.`企业名称`, a.`准入类别` ORDER BY a.`申请日期`, a.`准入类别`, a.`准入类别` " +
 				"limit " + start + "," +  end
 			_, err = s.DBE.Query(sql)
+			s.DBE.Query("commit;")
 		}
+
+		s.DBE.Query("SET unique_checks=1;")
+		s.DBE.Query("SET autocommit=1;")
 	}
 
 	return err
 }
 
 func (s *TmpzcgtSession) InsertOilInfoChange() error {
+	s.DBE.Query("SET unique_checks=0;")
+	s.DBE.Query("SET autocommit=0;")
 	sql := "INSERT INTO OilInfoChange (Status,AccessCardNo,SupplierName,OldSupplierName,SupplierTypeName,SupplierTypeCode,OilCertificateNo,Grade,OperType,Country,MaunAgent,ConstructTeam,CommercialNo,OrganCode,CountryTaxNo,LocalTaxNo, " +
 		"Address,ZipCode,QualitySystemCert,ProductQualityCert,MaunLicense,QualifCert,QualifCertLevel,SafetyLicense,TechServiceLic,TJInNotify, " +
 		"SpecIndustryCert,LegalPerson,CategoryCode,CategoryName,RegCapital,Currency,ContactName,CompanyType,DepositBank,BankAccount,EMail, " +
@@ -391,6 +428,9 @@ func (s *TmpzcgtSession) InsertOilInfoChange() error {
 		"开户银行 AS DepositBank, 银行账号 AS BankAccount, 电子邮箱 AS EMail, 银行信用等级 AS BankCreditRating, 移动电话 AS Mobile, 固定电话 AS Telphone, 传真 AS Fax, 公司电话 AS CompanyTel, QQ号码 AS QQ, 公司网址 AS CompanyUrl, " +
 		"申请变更原因及内容 AS Remark, 审核日期 as  CreateOn, 录入员 as CreateBy,ID AS HouseNo from T_变更信息 "
 	_, err := s.DBE.Query(sql)
+	s.DBE.Query("commit;")
+	s.DBE.Query("SET unique_checks=1;")
+	s.DBE.Query("SET autocommit=1;")
 	return err
 }
 
@@ -432,10 +472,18 @@ func (s *TmpzcgtSession) UpOilSupplierCertAppend() error {
 	/** sql := "UPDATE OilSupplierCertAppend b SET " +
 		"b.SupplierId = IFNULL((SELECT a.`SupplierId` FROM OilSupplierCert a WHERE a.`GfId` = b.OldId AND a.SupplierTypeCode = b.AppendType limit 1), 0), " +
 		"b.SupplierCertId = IFNULL((SELECT a.`Id` FROM OilSupplierCert a WHERE a.`GfId` = b.OldId AND a.SupplierTypeCode = b.AppendType limit 1), 0) " */
+	s.DBE.Query("SET unique_checks=0;")
+	s.DBE.Query("SET autocommit=0;")
+
 	sql := "UPDATE OilSupplierCertAppend b left join OilSupplierCert a ON (a.`GfId` = b.OldId AND a.SupplierTypeCode = b.AppendType) SET " +
 		"b.SupplierId = IFNULL(a.`SupplierId`, 0), " +
 		"b.SupplierCertId = IFNULL(a.`Id`, 0) "
 	_, err := s.DBE.Query(sql)
+
+	s.DBE.Query("commit;")
+
+	s.DBE.Query("SET unique_checks=1;")
+	s.DBE.Query("SET autocommit=1;")
 	return err
 }
 
@@ -457,6 +505,9 @@ func (s *TmpzcgtSession) UpdateOilSupplierCertAppendSub() error {
 	var certAppendList []suppliercertappend.OilSupplierCertAppend
 	svc.GetEntitysByWhere("OilSupplierCertAppend", "", &certAppendList)
 	var err error
+	s.DBE.Query("SET unique_checks=0;")
+	s.DBE.Query("SET autocommit=0;")
+	s.DBE.Query("ALTER TABLE `OilSupplierCertSub` ADD INDEX index_oldid ( `OldId` );")
 	for _, certAppend := range certAppendList {
 		appendCodeList := strings.Split(certAppend.Remark, ",")
 		for _, appendCode := range appendCodeList {
@@ -464,13 +515,19 @@ func (s *TmpzcgtSession) UpdateOilSupplierCertAppendSub() error {
 				", a.`Type`=3 WHERE a.SupplierCertId=" + strconv.Itoa(certAppend.SupplierCertId) +
 				" AND a.SupplierTypeCode LIKE '" + appendCode + "%' and SupplierCertAppendId=0"
 			_, err = s.DBE.Query(sql)
+			s.DBE.Query("commit;")
 		}
 	}
+	s.DBE.Query("alter table OilSupplierCertSub drop index index_oldid ;")
+	s.DBE.Query("SET unique_checks=1;")
+	s.DBE.Query("SET autocommit=1;")
 	return err
 }
 
 // 导入资质名称
 func (s *TmpzcgtSession) InsertFileName() error {
+	s.DBE.Query("SET unique_checks=0;")
+	s.DBE.Query("SET autocommit=0;")
 	sql := "INSERT INTO `OilSupplierFile` (SupplierId, SupplierCertSubId, Remark, NeedFileType, OtherRemark, SupplierTypeCode,CreateBy) " +
 		"SELECT 0 AS SupplierId, 0 AS SupplierCertSubId, z.企业名称 AS Remark ,f.类别 AS NeedFileType, f.说明 AS OtherRemark, " +
 		"( CASE 准入类别 WHEN '物资类' THEN '01' WHEN '基建类' THEN '02' WHEN '服务类' THEN '03' WHEN '技术服务类' THEN '03' else '000' END ) SupplierTypeCode, " +
@@ -479,6 +536,10 @@ func (s *TmpzcgtSession) InsertFileName() error {
 		"LEFT JOIN `T_资格预审表` z ON (f.ID_GF = z.ID) " +
 		"WHERE z.企业名称 IS NOT NULL "
 	_, err := s.DBE.Query(sql)
+
+	s.DBE.Query("commit;")
+	s.DBE.Query("SET unique_checks=1;")
+	s.DBE.Query("SET autocommit=1;")
 	return err
 }
 

src/dashoo.cn/backend/api/controllers/tmpzcgf/tmpzcgf.go

@@ -868,7 +868,7 @@ func (this *TmpzcgfController) InsertAll() {
 		this.UpdateOilSupplierCertAppend()
 		this.UpdateOilSupplierCertAppendSub() // 更新增项的准入范围
 
-		this.CapacityExpansion() // 编码扩充
+		// this.CapacityExpansion() // 编码扩充
 		// 历史数据导入-由于运行到此处中断，上面的代码临时注释
 		this.InsterAnnualAudit()
 		this.UpdateAnnualAudit()