Go 实现 AWS4 请求认证

什么是 Amazon S3？它是 AWS 提供的一种对象存储服务，提供行业领先的可扩展性、数据可用性、安全性和性能。Amazon S3 可达到 99.999999999%（11 个 9）的持久性。
Amazon S3 里用到了 AWS Signature Version 4（下面简称 AWS4）做认证请求，这篇文章将会讲解如何使用 Go 实现 AWS4 请求认证。

AWS4 是一种用于对所有 AWS 区域服务的入站 API 请求进行身份验证的协议。

AWS4

AWS4 对请求进行签名有以下优势（但这也取决于你如何使用）：

• 验证请求者的身份 - 经过身份验证的请求需要使用 AccessKeyID 和 SecretAccessKey 创建签名。
• 保护传输中的数据 - 为了防止在传输过程中对请求进行篡改，可以使用一些请求元素（比如请求路径、请求头等）来计算请求签名。Amazon S3 在收到请求后，使用相同的请求元素来计算签名。如果 Amazon S3 接收到的任何请求组件与用于计算签名的组件不匹配，Amazon S3 将拒绝该请求。
• 防止重用请求的签名部分 - 请求的签名部分在请求中的时间戳的一段时间内有效。

授权方式

• HTTP 身份验证头，例如 Authorization 请求头：

Authorization: AWS4-HMAC-SHA256 
Credential=AKIAIOSFODNN7EXAMPLE/20130524/us-east-1/s3/aws4_request, 
SignedHeaders=host;range;x-amz-date,
Signature=fe5f80f77d5fa3beca038a248ff027d0445342fe2855ddc963176630326f1024

• URL 查询字符串参数，例如预签名 URL：

https://s3.amazonaws.com/examplebucket/test.txt
?X-Amz-Algorithm=AWS4-HMAC-SHA256
&X-Amz-Credential=<your-access-key-id>/20130721/us-east-1/s3/aws4_request
&X-Amz-Date=20130721T201207Z
&X-Amz-Expires=86400
&X-Amz-SignedHeaders=host
&X-Amz-Signature=<signature-value>

Go 实现 HTTP 身份验证头

HTTP 身份验证头组成部分

• AWS4-HMAC-SHA256 - 该字符串指定 AWS4 和签名算法（HMAC-SHA256）。
• Credential - 指定 AccessKeyID、计算签名的日期、区域和服务。它的格式是<your-access-key-id>/<date>/<aws-region>/<aws-service>/aws4_request，date 的格式是 YYYYMMDD。
• SignedHeaders - 指定用于计算签名的请求头列表，以分号分隔。仅包含请求头的名称，且必须是小写，例如：host;range;x-amz-date。
• Signature - 表示为 64 个小写十六进制字符的 256 位签名，例如：fe5f80f77d5fa3beca038a248ff027d0445342fe2855ddc963176630326f1024。

上传图片至掘金

根据对 AWS4 请求认证的学习，并参考 simples3 的代码实现了 上传图片至掘金（字节跳动的存储服务，其服务名称是imagex）的功能，下面是部分的代码实现（主要省略了Client 部分的实现，完整代码有待后续完善后进行开源）：

package juejin

import (
    "bytes"
    "crypto/hmac"
    "crypto/sha256"
    "encoding/hex"
    "encoding/json"
    "fmt"
    "hash/crc32"
    "io"
    "io/ioutil"
    "net/http"
    "net/url"
    "os"
    "path/filepath"
    "regexp"
    "sort"
    "strings"
    "time"

    "github.com/tidwall/gjson"
)

const (
    amzDateISO8601TimeFormat = "20060102T150405Z"
    shortTimeFormat          = "20060102"
    algorithm                = "AWS4-HMAC-SHA256"
    serviceName              = "imagex"
    serviceID                = "k3u1fbpfcp"
    version                  = "2018-08-01"
    uploadURLFormat          = "https://%s/%s"

    RegionCNNorth = "cn-north-1"

    actionApplyImageUpload  = "ApplyImageUpload"
    actionCommitImageUpload = "CommitImageUpload"

    polynomialCRC32 = 0xEDB88320
)

var (
    newLine = []byte{'\n'}

    // if object matches reserved string, no need to encode them
    reservedObjectNames = regexp.MustCompile("^[a-zA-Z0-9-_.~/]+$")
)

type ImageX struct {
    AccessKey string
    SecretKey string
    Region    string
    Client    *http.Client

    Token   string
    Version string
    BaseURL string
}

type UploadToken struct {
    AccessKeyID     string `json:"AccessKeyID"`
    SecretAccessKey string `json:"SecretAccessKey"`
    SessionToken    string `json:"SessionToken"`
}

func (c *Client) UploadImage(region, imgPath string) (string, error) {
    uploadToken, err := c.GetUploadToken()
    if err != nil {
        return "", err
    }

    ix := &ImageX{
        AccessKey: uploadToken.AccessKeyID,
        SecretKey: uploadToken.SecretAccessKey,
        Token:     uploadToken.SessionToken,
        Region:    region,
    }

    applyRes, err := ix.ApplyImageUpload()
    if err != nil {
        return "", err
    }

    storeInfo := gjson.Get(applyRes, "Result.UploadAddress.StoreInfos.0")
    storeURI := storeInfo.Get("StoreUri").String()
    storeAuth := storeInfo.Get("Auth").String()
    uploadHost := gjson.Get(applyRes, "Result.UploadAddress.UploadHosts.0").String()
    uploadURL := fmt.Sprintf(uploadURLFormat, uploadHost, storeURI)
    if err := ix.Upload(uploadURL, imgPath, storeAuth); err != nil {
        return "", err
    }

    sessionKey := gjson.Get(applyRes, "Result.UploadAddress.SessionKey").String()
    if _, err = ix.CommitImageUpload(sessionKey); err != nil {
        return "", err
    }

    return c.GetImageURL(storeURI)
}

func (c *Client) GetImageURL(uri string) (string, error) {
    endpoint := "/imagex/get_img_url"
    params := &url.Values{
        "uri": []string{uri},
    }
    raw, err := c.Get(APIBaseURL, endpoint, params)
    if err != nil {
        return "", err
    }
    rawurl := gjson.Get(raw, "data.main_url").String()
    return rawurl, nil
}

func (c *Client) GetUploadToken() (*UploadToken, error) {
    endpoint := "/imagex/gen_token"
    params := &url.Values{
        "client": []string{"web"},
    }
    raw, err := c.Get(APIBaseURL, endpoint, params)
    if err != nil {
        return nil, err
    }
    var token *UploadToken
    err = json.Unmarshal([]byte(gjson.Get(raw, "data.token").String()), &token)
    return token, err
}

func (ix *ImageX) ApplyImageUpload() (string, error) {
    rawurl := fmt.Sprintf("https://imagex.bytedanceapi.com/?Action=%s&Version=%s&ServiceId=%s",
        actionApplyImageUpload, version, serviceID)
    req, err := http.NewRequest(http.MethodGet, rawurl, nil)
    if err != nil {
        return "", err
    }

    if err := ix.signRequest(req); err != nil {
        return "", err
    }

    res, err := ix.getClient().Do(req)
    if err != nil {
        return "", err
    }
    defer res.Body.Close()
    b, err := ioutil.ReadAll(res.Body)
    if err != nil {
        return "", err
    }
    raw := string(b)
    if res.StatusCode != 200 || gjson.Get(raw, "ResponseMetadata.Error").Exists() {
        return "", fmt.Errorf("raw: %s, response: %+v", raw, res)
    }
    return raw, nil
}

func (ix *ImageX) CommitImageUpload(sessionKey string) (string, error) {
    rawurl := fmt.Sprintf("https://imagex.bytedanceapi.com/?Action=%s&Version=%s&SessionKey=%s&ServiceId=%s",
        actionCommitImageUpload, version, sessionKey, serviceID)
    req, err := http.NewRequest(http.MethodPost, rawurl, nil)
    if err != nil {
        return "", err
    }

    if err := ix.signRequest(req); err != nil {
        return "", err
    }

    res, err := ix.getClient().Do(req)
    if err != nil {
        return "", err
    }
    defer res.Body.Close()
    b, err := ioutil.ReadAll(res.Body)
    if err != nil {
        return "", err
    }
    raw := string(b)
    if res.StatusCode != 200 || gjson.Get(raw, "ResponseMetadata.Error").Exists() {
        return "", fmt.Errorf("raw: %s, response: %+v", raw, res)
    }
    return raw, nil
}

func (ix *ImageX) getClient() *http.Client {
    if ix.Client == nil {
        return http.DefaultClient
    }
    return ix.Client
}

func (ix *ImageX) signKeys(t time.Time) []byte {
    h := makeHMac([]byte("AWS4"+ix.SecretKey), []byte(t.Format(shortTimeFormat)))
    h = makeHMac(h, []byte(ix.Region))
    h = makeHMac(h, []byte(serviceName))
    h = makeHMac(h, []byte("aws4_request"))
    return h
}

func (ix *ImageX) writeRequest(w io.Writer, r *http.Request) error {
    r.Header.Set("host", r.Host)

    w.Write([]byte(r.Method))
    w.Write(newLine)
    writeURI(w, r)
    w.Write(newLine)
    writeQuery(w, r)
    w.Write(newLine)
    writeHeader(w, r)
    w.Write(newLine)
    w.Write(newLine)
    writeHeaderList(w, r)
    w.Write(newLine)
    return writeBody(w, r)
}

func (ix *ImageX) writeStringToSign(w io.Writer, t time.Time, r *http.Request) error {
    w.Write([]byte(algorithm))
    w.Write(newLine)
    w.Write([]byte(t.Format(amzDateISO8601TimeFormat)))
    w.Write(newLine)

    w.Write([]byte(ix.creds(t)))
    w.Write(newLine)

    h := sha256.New()
    if err := ix.writeRequest(h, r); err != nil {
        return err
    }
    fmt.Fprintf(w, "%x", h.Sum(nil))
    return nil
}

func (ix *ImageX) creds(t time.Time) string {
    return t.Format(shortTimeFormat) + "/" + ix.Region + "/" + serviceName + "/aws4_request"
}

func (ix *ImageX) signRequest(req *http.Request) error {
    t := time.Now().UTC()
    req.Header.Set("x-amz-date", t.Format(amzDateISO8601TimeFormat))

    req.Header.Set("x-amz-security-token", ix.Token)

    k := ix.signKeys(t)
    h := hmac.New(sha256.New, k)

    if err := ix.writeStringToSign(h, t, req); err != nil {
        return err
    }

    auth := bytes.NewBufferString(algorithm)
    auth.Write([]byte(" Credential=" + ix.AccessKey + "/" + ix.creds(t)))
    auth.Write([]byte{',', ' '})
    auth.Write([]byte("SignedHeaders="))
    writeHeaderList(auth, req)
    auth.Write([]byte{',', ' '})
    auth.Write([]byte("Signature=" + fmt.Sprintf("%x", h.Sum(nil))))

    req.Header.Set("authorization", auth.String())
    return nil
}

func writeURI(w io.Writer, r *http.Request) {
    path := r.URL.RequestURI()
    if r.URL.RawQuery != "" {
        path = path[:len(path)-len(r.URL.RawQuery)-1]
    }
    slash := strings.HasSuffix(path, "/")
    path = filepath.Clean(path)
    if path != "/" && slash {
        path += "/"
    }
    w.Write([]byte(path))
}
func writeQuery(w io.Writer, r *http.Request) {
    var a []string
    for k, vs := range r.URL.Query() {
        k = url.QueryEscape(k)
        for _, v := range vs {
            if v == "" {
                a = append(a, k)
            } else {
                v = url.QueryEscape(v)
                a = append(a, k+"="+v)
            }
        }
    }
    sort.Strings(a)
    for i, s := range a {
        if i > 0 {
            w.Write([]byte{'&'})
        }
        w.Write([]byte(s))
    }
}

func writeHeader(w io.Writer, r *http.Request) {
    i, a := 0, make([]string, len(r.Header))
    for k, v := range r.Header {
        sort.Strings(v)
        a[i] = strings.ToLower(k) + ":" + strings.Join(v, ",")
        i++
    }
    sort.Strings(a)
    for i, s := range a {
        if i > 0 {
            w.Write(newLine)
        }
        io.WriteString(w, s)
    }
}

func writeHeaderList(w io.Writer, r *http.Request) {
    i, a := 0, make([]string, len(r.Header))
    for k := range r.Header {
        a[i] = strings.ToLower(k)
        i++
    }
    sort.Strings(a)
    for i, s := range a {
        if i > 0 {
            w.Write([]byte{';'})
        }
        w.Write([]byte(s))
    }
}

func writeBody(w io.Writer, r *http.Request) error {
    var (
        b   []byte
        err error
    )
    // If the payload is empty, use the empty string as the input to the SHA256 function
    // http://docs.amazonwebservices.com/general/latest/gr/sigv4-create-canonical-request.html
    if r.Body == nil {
        b = []byte("")
    } else {
        b, err = ioutil.ReadAll(r.Body)
        if err != nil {
            return err
        }
        r.Body = ioutil.NopCloser(bytes.NewBuffer(b))
    }

    h := sha256.New()
    h.Write(b)
    fmt.Fprintf(w, "%x", h.Sum(nil))
    return nil
}

func makeHMac(key []byte, data []byte) []byte {
    hash := hmac.New(sha256.New, key)
    hash.Write(data)
    return hash.Sum(nil)
}

func (ix *ImageX) Upload(rawurl, fp, auth string) error {
    crc32, err := hashFileCRC32(fp)
    if err != nil {
        return err
    }
    file, err := os.Open(fp)
    if err != nil {
        return err
    }
    defer file.Close()

    req, err := http.NewRequest(http.MethodPost, rawurl, file)
    if err != nil {
        return err
    }
    req.Header.Add("authorization", auth)
    req.Header.Add("Content-Type", "application/octet-stream")
    req.Header.Add("content-crc32", crc32)
    res, err := http.DefaultClient.Do(req)
    if err != nil {
        return err
    }
    defer res.Body.Close()
    b, err := ioutil.ReadAll(res.Body)
    if err != nil {
        return err
    }
    raw := string(b)
    if gjson.Get(raw, "success").Int() != 0 {
        return fmt.Errorf("raw: %s, response: %+v", raw, res)
    }
    return nil
}

// hashFileCRC32 generate CRC32 hash of a file
// Refer https://mrwaggel.be/post/generate-crc32-hash-of-a-file-in-golang-turorial/
func hashFileCRC32(filePath string) (string, error) {
    file, err := os.Open(filePath)
    if err != nil {
        return "", err
    }
    defer file.Close()
    tablePolynomial := crc32.MakeTable(polynomialCRC32)
    hash := crc32.New(tablePolynomial)
    if _, err := io.Copy(hash, file); err != nil {
        return "", err
    }
    return hex.EncodeToString(hash.Sum(nil)), nil
}

转自：https://juejin.cn/post/6950300506946273294