add md5 and use b64 for digest encodings

actions · robherley · May 19, 2022 · Apr 4, 2022 · Apr 5, 2022 · Apr 26, 2022
commit a3c696e88e8572ce57e213ed63921bdd2fdb211e
@@ -1,23 +1,36 @@
-import CRC64 from '../src/internal/crc64'
+import CRC64, {CRC64DigestEncoding} from '../src/internal/crc64'
 
 const fixtures = {
   data:
     '🚀 👉😎👉 Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.\nUt enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat.\nDuis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur.\nExcepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.\n',
-  expected: '846CE4ADAD6223ED'
+  expected: {
+    hex: '846CE4ADAD6223ED',
+    base64: '7SNira3kbIQ=',
+    buffer: Buffer.from([0xed, 0x23, 0x62, 0xad, 0xad, 0xe4, 0x6c, 0x84])
+  }
+}
+
+function assertEncodings(crc: CRC64): void {
+  const encodings = Object.keys(fixtures.expected) as CRC64DigestEncoding[]
+  for (const encoding of encodings) {
+    expect(crc.digest(encoding)).toEqual(fixtures.expected[encoding])
+  }
 }
 
 describe('@actions/artifact/src/internal/crc64', () => {
   it('CRC64 from string', async () => {
     const crc = new CRC64()
     crc.update(fixtures.data)
-    expect(crc.digest()).toEqual(fixtures.expected)
+
+    assertEncodings(crc)
   })
 
   it('CRC64 from buffer', async () => {
     const crc = new CRC64()
     const buf = Buffer.from(fixtures.data)
     crc.update(buf)
-    expect(crc.digest()).toEqual(fixtures.expected)
+
+    assertEncodings(crc)
   })
 
   it('CRC64 from split data', async () => {
@@ -26,7 +39,8 @@ describe('@actions/artifact/src/internal/crc64', () => {
     for (const split of splits) {
       crc.update(`${split}\n`)
     }
-    expect(crc.digest()).toEqual(fixtures.expected)
+
+    assertEncodings(crc)
   })
 
   it('flips 64 bits', async () => {

@@ -75,7 +75,10 @@ describe('Utils', () => {
     const size = 24
     const uncompressedLength = 100
     const range = 'bytes 0-199/200'
-    const digest = 'FFFCD6894DC82C6D'
+    const digest = {
+      crc64: 'bSzITYnW/P8=',
+      md5: 'Xiv1fT9AxLbfadrxk2y3ZvgyN0tPwCWafL/wbi9w8mk='
+    }
     const headers = utils.getUploadHeaders(
       contentType,
       true,
@@ -85,7 +88,7 @@ describe('Utils', () => {
       range,
       digest
     )
-    expect(Object.keys(headers).length).toEqual(9)
+    expect(Object.keys(headers).length).toEqual(10)
     expect(headers['Accept']).toEqual(
       `application/json;api-version=${utils.getApiVersion()}`
     )
@@ -96,7 +99,8 @@ describe('Utils', () => {
     expect(headers['x-tfs-filelength']).toEqual(uncompressedLength)
     expect(headers['Content-Length']).toEqual(size)
     expect(headers['Content-Range']).toEqual(range)
-    expect(headers['x-actions-result-crc64']).toEqual(digest)
+    expect(headers['x-actions-results-crc64']).toEqual(digest.crc64)
+    expect(headers['x-actions-results-md5']).toEqual(digest.md5)
   })
 
   it('Test constructing upload headers with only required parameter', () => {
@@ -229,6 +233,7 @@ describe('Utils', () => {
     const stream = Readable.from(data)
     const digest = await utils.digestForStream(stream)
 
-    expect(digest).toBe('FFFCD6894DC82C6D')
+    expect(digest.crc64).toBe('bSzITYnW/P8=')
+    expect(digest.md5).toBe('Xiv1fT9AxLbfadrxk2y3ZvgyN0tPwCWafL/wbi9w8mk=')
   })
 })
@@ -269,6 +269,8 @@ const PREGEN_POLY_TABLE = [
   BigInt('0x2ADA5047EFEC8728')
 ]
 
+export type CRC64DigestEncoding = 'hex' | 'base64' | 'buffer'
+
 class CRC64 {
   private _crc: bigint
 
@@ -288,8 +290,23 @@ class CRC64 {
     this._crc = CRC64.flip64Bits(crc)
   }
 
-  digest(): string {
-    return this._crc.toString(16).toUpperCase()
+  digest(encoding?: CRC64DigestEncoding): string | Buffer {
+    switch (encoding) {
+      case 'hex':
+        return this._crc.toString(16).toUpperCase()
+      case 'base64':
+        return this.toBuffer().toString('base64')
+      default:
+        return this.toBuffer()
+    }
+  }
+
+  private toBuffer(): Buffer {
+    return Buffer.from(
+      [0, 8, 16, 24, 32, 40, 48, 56].map(s =>
+        Number((this._crc >> BigInt(s)) & BigInt(0xff))
+      )
+    )
   }
 
   static flip64Bits(n: bigint): bigint {

@@ -1,3 +1,4 @@
+import crypto from 'crypto'
 import {promises as fs} from 'fs'
 import {IncomingHttpHeaders} from 'http'
 import {debug, info, warning} from '@actions/core'
@@ -182,7 +183,7 @@ export function getUploadHeaders(
   uncompressedLength?: number,
   contentLength?: number,
   contentRange?: string,
-  digest?: string
+  digest?: StreamDigest
 ): IHeaders {
   const requestOptions: IHeaders = {}
   requestOptions['Accept'] = `application/json;api-version=${getApiVersion()}`
@@ -205,7 +206,8 @@ export function getUploadHeaders(
     requestOptions['Content-Range'] = contentRange
   }
   if (digest) {
-    requestOptions['x-actions-result-crc64'] = digest
+    requestOptions['x-actions-results-crc64'] = digest.crc64
+    requestOptions['x-actions-results-md5'] = digest.md5
   }
 
   return requestOptions
@@ -297,13 +299,28 @@ export async function sleep(milliseconds: number): Promise<void> {
   return new Promise(resolve => setTimeout(resolve, milliseconds))
 }
 
+export interface StreamDigest {
+  crc64: string
+  md5: string
+}
+
 export async function digestForStream(
   stream: NodeJS.ReadableStream
-): Promise<string> {
+): Promise<StreamDigest> {
   return new Promise((resolve, reject) => {
-    const hasher = new CRC64()
-    stream.on('data', data => hasher.update(data))
-    stream.on('end', () => resolve(hasher.digest()))
-    stream.on('error', reject)
+    const crc64 = new CRC64()
+    const md5 = crypto.createHash('sha256')
+    stream
+      .on('data', data => {
+        crc64.update(data)
+        md5.update(data)
+      })
+      .on('end', () =>
+        resolve({
+          crc64: crc64.digest('base64') as string,
+          md5: md5.digest('base64')
+        })
+      )
+      .on('error', reject)
   })
 }