客服电话
APP下载
迪恩网络APP
随时随地掌握行业动态
官方微信
扫描二维码
关注迪恩网络微信公众号
优化版本想看优化版本请移步： 使用Go解析HTTP返回数据为struct并存入数据库的操作 前言 新项目使用Go搭建服务，其中涉及到很多业务数据的构建以及处理的逻辑，笔者也是刚刚开始写Go代码，刚刚开始的时候必然会踩很多坑，这里就记录一下笔者在处理SDK返回的层级数据时遇到的问题以及后续的优化处理。 业务需求描述从某平台获取到的HTTP原始数据格式如下所示： { "request_status": "SUCCESS", "request_id": "xxxxxx", "paging": {}, "adaccounts": [ { "sub_request_status": "SUCCESS", "adaccount": { "id": "xxx", "updated_at": "2020-10-28T22:09:24.409Z", "created_at": "2020-08-21T11:00:28.455Z", "name": "xxx", "type": "PARTNER", "status": "ACTIVE", "organization_id": "xxx", "funding_source_ids": [ "xxx" ], "currency": "USD", "timezone": "Asia/Shanghai", } }, { "sub_request_status": "SUCCESS", "adaccount": { "id": "xxx", "updated_at": "2020-10-28T21:50:52.923Z", "created_at": "2020-08-21T10:59:07.409Z", "name": "xxx", "type": "PARTNER", "status": "ACTIVE", "organization_id": "xxx", "funding_source_ids": [ "xxx", "fc7cb056-453a-4b3f-8294-xxx" ], "currency": "USD", "timezone": "Asia/Shanghai", } }, { "sub_request_status": "SUCCESS", "adaccount": { "id": "47ea8129-xxx-xxx-xxx-xxx", "updated_at": "2020-10-28T21:57:05.953Z", "created_at": "2020-08-21T10:57:34.614Z", "name": "xxx", "type": "PARTNER", "status": "ACTIVE", "organization_id": "16412453-xxx-xxx-xxx-xxx", "funding_source_ids": [ "xxx-xxx-4b3f-8294-xxx" ], "currency": "USD", "timezone": "Asia/Shanghai", } }, { "sub_request_status": "SUCCESS", "adaccount": { "id": "xxx-ed2a-xxx-xxx-xxx", "updated_at": "2020-10-28T21:56:38.374Z", "created_at": "2020-08-21T10:58:35.585Z", "name": "xxx", "type": "PARTNER", "status": "ACTIVE", "organization_id": "xxx-e008-4353-xxx-xxx", "funding_source_ids": [ "xxx-453a-xxx-8294-xxx" ], "currency": "USD", "timezone": "Asia/Shanghai", } }, { "sub_request_status": "SUCCESS", "adaccount": { "id": "xxx-fe6c-xxx-a606-xxx", "updated_at": "2020-10-28T21:56:34.531Z", "created_at": "2020-08-21T10:59:53.850Z", "name": "Fashowtime_04_Muyou_EC_SINO_B", "type": "PARTNER", "status": "ACTIVE", "organization_id": "xxx-e008-4353-xxx-xxx", "funding_source_ids": [ "fc7cb056-xxx-4b3f-xxx-xxx" ], "currency": "USD", "timezone": "Asia/Shanghai", } } ] } 现在想要将这些数据分别存入MySQL数据库中。 数据库的表结构如下：
第一版实现思路思路一开始，我的思路是：先将这些数据转换为多层嵌套的map结构，然后遍历这个结果，将结果构建成一个切片，这个切片中存放的是只有一层结构的每个账号的数据，最后遍历这个切片，针对切片中每个账号的数据进行处理（还需要将map转换为struct）存入数据库中。 代码第一版的代码先发出来： package main import ( "encoding/json" "errors" "fmt" "github.com/jinzhu/gorm" _ "github.com/jinzhu/gorm/dialects/mysql" "time" "io/ioutil" "log" "net/http" ) const ( BINano = "2006-01-02 15:04:05.000000000" BIMicro = "2006-01-02 15:04:05.000000" BIMil = "2006-01-02 15:04:05.000" BISec = "2006-01-02 15:04:05" BICST = "2006-01-02 15:04:05 +0800 CST" BIUTC = "2006-01-02 15:04:05 +0000 UTC" BIDate = "2006-01-02" BITime = "15:04:05" ) type AdAccount struct { ID string `json:"id"` Name string `json:"name"` Status string `json:"status"` CreatedAt string `json:"created_at"` CreatedAtTimeStmp int64 `json:"-"` } // 设置表名为 adaccount func (AdAccount) TableName() string { return "adaccount" } // 需要用到的字段 var accountArr = []string{"id", "name", "status", "created_at"} func main() { // 初始化db db, err := gorm.Open("mysql", "root:123@(localhost)/gorm1?charset=utf8mb4&parseTime=True&loc=Local") if err != nil { fmt.Println("err>>> ", err) // panic panic(err) } // defer defer db.Close() // 自动迁移 建表等。。。 db.AutoMigrate(&AdAccount{}) // 刷新token TODO：后续放在Redis中优化一下 token := refreshToken() //fmt.Println("token>>>>>", token) //token := "xxx" // TODO: 账号信息 先用一个固定的 organization 编号写代码 adAccounts := getOrganAdAccounts(baseRqeuest, "xx-e008-xx-xx-xx", token) fmt.Println("adAccounts>>>>>", adAccounts, len(adAccounts)) for _, currM := range adAccounts{ fmt.Printf("%T, %v \n",currM,currM) // map[string]interface {}, map[id:xxx name:xxx status:ACTIVE] // currM是一个interface得转一下才能取值！！！ changeM := currM.(map[string]interface{}) fmt.Printf("time1>>>>> %T, %v \n",changeM["created_at"],changeM["created_at"]) created_at := changeM["created_at"].(string) // 将时间字符串转换为时间戳 int64 created_at_int,_ := Timestr2Timestamp(created_at) changeM["created_at"] = created_at_int fmt.Printf("time2>>>>> %T, %v \n",changeM["created_at"],changeM["created_at"]) fmt.Print("changeM>>> ",changeM) // 然后将map转换为结构体 最终存入数据库中。。。。。。 } } // 判断字符串是否在切片中 func IsContain(strList []string, item string) bool { for _, str := range strList { if str == item { return true } } return false } func getOrganAdAccounts(f func(...string) map[string]interface{}, organizationID string, token string) []interface{} { requestMethod := "GET" url := "https://adsapi.snapchat.com/v1/organizations/" + organizationID + "/adaccounts" ret := f(token, requestMethod, url) //result := NewAdAccountFromJsonString(ret) result := handleHttpRequest(ret, "adaccount") return result } // base func baseRqeuest(args ...string) map[string]interface{} { // token requestMethod url 三个参数是有顺序的！ token := args[0] requestMethod := args[1] url := args[2] client := &http.Client{} // get请求 req, err := http.NewRequest(requestMethod, url, nil) if err != nil { fmt.Println(err) log.Fatal(err) } // 在请求头中加入校验的token req.Header.Set("Authorization", "Bearer "+token) resp, err := client.Do(req) if err != nil { fmt.Println(err) log.Fatal(err) } returnMap, err := ParseResponse(resp) return returnMap } // 解析http请求返回的内容 func ParseResponse(response *http.Response) (map[string]interface{}, error) { var result map[string]interface{} body, err := ioutil.ReadAll(response.Body) // 将 body io数据流转换为 map[string]interface{} 类型返回！ if err == nil { err = json.Unmarshal(body, &result) } return result, err } // refresh token func refreshToken() string { client := &http.Client{} refreshToken := "xxxxxudoucxC2uZInEyQ" clientId := "xxx" clientSecret := "xxx" grantType := "refresh_token" // 字符串拼接 var queryData string queryData = fmt.Sprint("?refresh_token=", refreshToken, "&client_id=", clientId, "&client_secret=", clientSecret, "&grant_type=", grantType) url := "https://accounts.snapchat.com/login/oauth2/access_token" + queryData req, err := http.NewRequest("POST", url, nil) if err != nil { log.Fatal(err) } resp, err := client.Do(req) if err != nil { log.Fatal(err) } // 解析 ret, err := ParseResponse(resp) // 转为string再返回 token := ret["access_token"].(string) return token } // 处理结构化的数据 返回存空接口的切片：里面可以存储任何类型的数据，方便做批量处理 func handleHttpRequest(httpRet map[string]interface{}, handleType string) (result []interface{}) { // 外层的key比里层的key多一个s handleTypes := handleType + "s" for key, val := range httpRet { if key == handleTypes { //fmt.Println(666) mp := val.([]interface{}) // 遍历最外层 for _, orga_val := range mp { fmt.Println(": orga_val>>>>", orga_val) // 判断type取数 switch orga_val.(type) { // 如果是一个字典，继续获取里面的值 case map[string]interface{}: // 转换完后再遍历 orgaValNew := orga_val.(map[string]interface{}) // 遍历 +s 的那个列表 fmt.Println("==========================================================================") for dicKey, dicVal := range orgaValNew { //fmt.Println(dicKey,"<><><>") // 是对应的key才取值! if dicKey == handleType { switch dicVal.(type) { case map[string]interface{}: // 转换完后再遍历 innerDic := dicVal.(map[string]interface{}) // 注意这里必须用临时的map存储要返回的每一个字典！！！ currentMap := make(map[string]interface{}) // 遍历列表中的每一个字典 for innerKey, innerVal := range innerDic { //fmt.Println(innerKey,"-------->",innerVal) // 获取不同类型的结果 // organization直接返回列表 if handleType == "organization" { if innerKey == "id" { id := innerVal.(string) result = append(result, id) } } else if handleType == "adaccount" { // 获取账户信息 flag := IsContain(accountArr, innerKey) if flag == true { //print("adaccount>>>>>>>\n") //id := innerVal currentMap[innerKey] = innerVal } } } // 构建结果 result = append(result, currentMap) } } } } } } } return } // 时间字符串转时间戳 func Timestr2Timestamp(str string) (int64, error) { return Timestr2TimestampBasic(str, "", nil) } func Timestr2TimestampBasic(str string, format string, loc *time.Location) (int64, error) { t, err := Timestr2TimeBasic(str, format, loc) if err != nil { return -1., err } return (int64(t.UnixNano()) * 1) / 1e9, nil } func Timestr2TimeBasic(value string, resultFormat string, resultLoc *time.Location) (time.Time, error) { /** - params value: 转换内容字符串 resultFormat: 结果时间格式 resultLoc: 结果时区 */ resultLoc = getLocationDefault(resultLoc) useFormat := []string{ // 可能的转换格式 BINano, BIMicro, BIMil, BISec, BICST, BIUTC, BIDate, BITime, time.RFC3339, time.RFC3339Nano, } var t time.Time for _, usef := range useFormat { tt, error := time.ParseInLocation(usef, value, resultLoc) t = tt if error != nil { continue } break } if t == getTimeDefault(resultLoc) { return t, errors.New("时间字符串格式错误") } if resultFormat == "" { resultFormat = "2006-01-02 15:04:05" } st := t.Format(resultFormat) fixedt, _ := time.ParseInLocation(resultFormat, st, resultLoc) return fixedt, nil } // 获取time默认值, 造一个错误 func getTimeDefault(loc *time.Location) time.Time { loc = getLocationDefault(loc) t, _ := time.ParseInLocation("2006-01-02 15:04:05", "", loc) return t } func getLocationDefault(loc *time.Location) *time.Location { if loc == nil { loc, _ = time.LoadLocation("Local") } return loc } map转struct的代码map转struct的代码参考我这篇博客：各种结构相互转换 重要功能说明请求及解析http的方法请求及解析http的方法如下（注意我是将解析后的结果转成了 map[string]interface{}），其中还用到了将函数作为参数的传参方式： func getOrganAdAccounts(f func(...string) map[string]interface{}, organizationID string, token string) []interface{} { requestMethod := "GET" url := "https://adsapi.snapchat.com/v1/organizations/" + organizationID + "/adaccounts" ret := f(token, requestMethod, url) //result := NewAdAccountFromJsonString(ret) result := handleHttpRequest(ret, "adaccount") return result } // base func baseRqeuest(args ...string) map[string]interface{} { // token requestMethod url 三个参数是有顺序的！ token := args[0] requestMethod := args[1] url := args[2] client := &http.Client{} // get请求 req, err := http.NewRequest(requestMethod, url, nil) 全部评论
专题导读
上一篇：适合新手：手把手教你用Go快速搭建高性能、可扩展的IM系统(有源码) ...发布时间：2022-07-10下一篇：Excelize 2.3.1 发布，Go 语言 Excel 文档基础库，支持加密表格文档 ...发布时间：2022-07-10热门推荐
热门话题
阅读排行榜
|
请发表评论