diff --git a/ChangeLog.md b/ChangeLog.md index 900ec37a2..6ec9a07d4 100644 --- a/ChangeLog.md +++ b/ChangeLog.md @@ -27,6 +27,7 @@ Blob: - Fixed service- and container-level Filter Blobs requests failing when the optional `where` query parameter is omitted. - Fixed blob operations hanging when a client disconnects before the operation queue processes the request. (issue #2575) - Implement `PutBlobFromUrl` (`Put Blob From URL`), which previously returned 501. The source is fetched over loopback, as `PutBlockFromURL` already does, so that SAS authentication and the `x-ms-source-if-*` conditions are enforced by the existing download path. Standard blob properties are copied from the source unless `x-ms-copy-source-blob-properties` is false, request blob content headers override them either way, request metadata replaces the source's rather than adding to it, and `x-ms-copy-source-tag-option: COPY` reads the source's tags over that same authorized path. An `x-ms-source-content-md5`, `x-ms-blob-content-md5`, `Content-MD5`, or `x-ms-content-crc64` header is checked against the copied content, and the response reports the MD5 and CRC64 of that content. A SAS needs Create or Write to create the blob, Write to overwrite it, and Tag as well when the request sets tags with `x-ms-tags` or copies the source's. As with `CopyBlobFromURL`, only sources on the same Azurite instance are supported. +- Fixed `PutBlobFromUrl` and `PutBlockFromURL` ignoring `x-ms-copy-source-authorization`. The header is now sent as the authorization of the copy source read, including the tags read for `x-ms-copy-source-tag-option: COPY`, so a bearer token authorizes a private source as on Azure. Bearer scheme casing and multiple separator spaces are normalized before the source read. Only the bearer scheme is accepted; any other scheme fails with `InvalidHeaderValue`. (issue #2815) Queue: diff --git a/src/blob/handlers/BlockBlobHandler.ts b/src/blob/handlers/BlockBlobHandler.ts index 43d28fd0d..39c709b51 100644 --- a/src/blob/handlers/BlockBlobHandler.ts +++ b/src/blob/handlers/BlockBlobHandler.ts @@ -6,6 +6,7 @@ import { TLSSocket } from "tls"; import { Readable } from "stream"; import { IExtentChunk } from "../../common/persistence/IExtentStore"; +import { BEARER_TOKEN_PREFIX } from "../../common/utils/constants"; import { convertRawHeadersToMetadata, getMD5FromString, @@ -268,15 +269,23 @@ export default class BlockBlobHandler containerName ); + const sourceAuthorization = BlockBlobHandler.sourceAuthorizationHeaders( + context, + options.copySourceAuthorization + ); + // Put Blob From URL always copies the whole source, so no range rides // along with the conditions. const sourceResponse = await this.readCopySource( context, "putBlobFromUrl", copySource, - BlockBlobHandler.sourceConditionHeaders( - options.sourceModifiedAccessConditions - ) + { + ...BlockBlobHandler.sourceConditionHeaders( + options.sourceModifiedAccessConditions + ), + ...sourceAuthorization + } ); // The status was only the response headers arriving; the body can still @@ -328,7 +337,11 @@ export default class BlockBlobHandler // came over, so a source that the caller may read but not tag refuses // the copy rather than leaking them. const blobTags = copySourceTags - ? await this.readCopySourceTags(context, copySource) + ? await this.readCopySourceTags( + context, + copySource, + sourceAuthorization + ) : options.blobTagsString === undefined ? undefined : getTagsFromString(options.blobTagsString, context.contextId!); @@ -595,6 +608,10 @@ export default class BlockBlobHandler headers, BlockBlobHandler.sourceConditionHeaders( options.sourceModifiedAccessConditions + ), + BlockBlobHandler.sourceAuthorizationHeaders( + context, + options.copySourceAuthorization ) ); @@ -960,6 +977,43 @@ export default class BlockBlobHandler return headers; } + /** + * Send the credential a copy request names for its source in + * x-ms-copy-source-authorization as the Authorization header of the + * source read, so that the token authenticator judges it as it would a + * client reading the source itself. + * + * Azure only accepts the Bearer scheme here. Any other scheme is refused + * rather than forwarded, since the source read would otherwise accept a + * shared key signature for it. + * + * @private + * @param {Context} context + * @param {string} [copySourceAuthorization] + * @returns {{ [key: string]: string }} + * @memberof BlockBlobHandler + */ + private static sourceAuthorizationHeaders( + context: Context, + copySourceAuthorization?: string + ): { [key: string]: string } { + if (!copySourceAuthorization) { + return {}; + } + // The token authenticator expects "Bearer" and one separator space. + const scheme = copySourceAuthorization.split(/\s/, 1)[0]; + if (scheme.toLowerCase() !== "bearer") { + throw StorageErrorFactory.getInvalidHeaderValue(context.contextId, { + HeaderName: "x-ms-copy-source-authorization" + }); + } + return { + authorization: `${BEARER_TOKEN_PREFIX}${copySourceAuthorization.slice( + scheme.length + ).replace(/^ +/, " ")}` + }; + } + /** * Read a copy source with a loopback self-request, so that SAS * authentication, ranges, and source conditions are answered by the @@ -976,7 +1030,8 @@ export default class BlockBlobHandler * @param {Context} context * @param {string} operation Handler method name, for log messages * @param {string} copySource The source URL the request named - * @param {{ [key: string]: string }} headers Conditions, ranges + * @param {{ [key: string]: string }} headers Conditions, ranges, and + * the source authorization * @param {string} [subresource] A query to append, such as "comp=tags" * @returns {Promise} A response whose body is a stream * @memberof BlockBlobHandler @@ -1110,23 +1165,26 @@ export default class BlockBlobHandler * Read the tags of a copy source, for the copy that asks to carry them * over. Real Azure charges this to the caller as its own Get Blob Tags * request against the source, and so does this: the authorization the - * source URL carries has to allow reading them. + * source URL or the source authorization header carries has to allow + * reading them. * * @private * @param {Context} context * @param {string} copySource The source URL the request named + * @param {{ [key: string]: string }} headers The source authorization * @returns {Promise} * @memberof BlockBlobHandler */ private async readCopySourceTags( context: Context, - copySource: string + copySource: string, + headers: { [key: string]: string } ): Promise { const response = await this.readCopySource( context, "putBlobFromUrl", copySource, - {}, + headers, "comp=tags" ); diff --git a/tests/blob/oauth.test.ts b/tests/blob/oauth.test.ts index 76f6e1040..28f204c5b 100644 --- a/tests/blob/oauth.test.ts +++ b/tests/blob/oauth.test.ts @@ -3,6 +3,7 @@ import { AccountSASPermissions, AccountSASResourceTypes, AccountSASServices, Ano import * as assert from "assert"; import { configLogger } from "../../src/common/Logger"; +import { computeHMACSHA256 } from "../../src/common/utils/utils"; import BlobTestServerFactory from "../BlobTestServerFactory"; import { EMULATOR_ACCOUNT_KEY, EMULATOR_ACCOUNT_NAME, generateJWTToken, getUniqueName } from "../testutils"; import { SimpleTokenCredential } from "../simpleTokenCredential"; @@ -937,6 +938,488 @@ describe("Blob OAuth Basic", () => { } }); + it(`Should authorize the copy source of Put Blob From URL with x-ms-copy-source-authorization @loki @sql`, async () => { + const token = generateJWTToken( + new Date("2019/01/01"), + new Date("2019/01/01"), + new Date("2100/01/01"), + "https://sts.windows-ppe.net/ab1f708d-50f6-404c-a006-d71b2ac7a606/", + "https://storage.azure.com", + "user_impersonation", + "23657296-5cd5-45b0-a809-d972a7f4dfe1", + "dd0d0df1-06c3-436c-8034-4b9a153097ce" + ); + + const serviceClient = new BlobServiceClient( + baseURL, + newPipeline(new SimpleTokenCredential(token), { + retryOptions: { maxTries: 1 }, + // Make sure socket is closed once the operation is done. + keepAliveOptions: { enable: false } + }) + ); + + const containerName: string = getUniqueName("1container-with-dash"); + const containerClient = serviceClient.getContainerClient(containerName); + await containerClient.create(); + + const content = "Hello World"; + const sourceClient = containerClient.getBlockBlobClient( + getUniqueName("source") + ); + await sourceClient.upload(content, content.length); + + const targetClient = containerClient.getBlockBlobClient( + getUniqueName("target") + ); + await targetClient.syncUploadFromURL(sourceClient.url, { + sourceAuthorization: { scheme: "Bearer", value: token } + }); + + const downloaded = await targetClient.downloadToBuffer(); + assert.deepStrictEqual(downloaded.toString(), content); + + await containerClient.delete(); + }); + + it(`Should authorize the copy source tags of Put Blob From URL with x-ms-copy-source-authorization @loki @sql`, async () => { + const token = generateJWTToken( + new Date("2019/01/01"), + new Date("2019/01/01"), + new Date("2100/01/01"), + "https://sts.windows-ppe.net/ab1f708d-50f6-404c-a006-d71b2ac7a606/", + "https://storage.azure.com", + "user_impersonation", + "23657296-5cd5-45b0-a809-d972a7f4dfe1", + "dd0d0df1-06c3-436c-8034-4b9a153097ce" + ); + + const serviceClient = new BlobServiceClient( + baseURL, + newPipeline(new SimpleTokenCredential(token), { + retryOptions: { maxTries: 1 }, + // Make sure socket is closed once the operation is done. + keepAliveOptions: { enable: false } + }) + ); + + const containerName: string = getUniqueName("1container-with-dash"); + const containerClient = serviceClient.getContainerClient(containerName); + await containerClient.create(); + + const content = "Hello World"; + const tags = { sourcetag: "sourcevalue" }; + const sourceClient = containerClient.getBlockBlobClient( + getUniqueName("source") + ); + await sourceClient.upload(content, content.length, { tags }); + + const targetClient = containerClient.getBlockBlobClient( + getUniqueName("target") + ); + await targetClient.syncUploadFromURL(sourceClient.url, { + copySourceTags: "COPY", + sourceAuthorization: { scheme: "Bearer", value: token } + }); + + const result = await targetClient.getTags(); + assert.deepStrictEqual(result.tags, tags); + + await containerClient.delete(); + }); + + it(`Should authorize the copy source of Put Block From URL with x-ms-copy-source-authorization @loki @sql`, async () => { + const token = generateJWTToken( + new Date("2019/01/01"), + new Date("2019/01/01"), + new Date("2100/01/01"), + "https://sts.windows-ppe.net/ab1f708d-50f6-404c-a006-d71b2ac7a606/", + "https://storage.azure.com", + "user_impersonation", + "23657296-5cd5-45b0-a809-d972a7f4dfe1", + "dd0d0df1-06c3-436c-8034-4b9a153097ce" + ); + + const serviceClient = new BlobServiceClient( + baseURL, + newPipeline(new SimpleTokenCredential(token), { + retryOptions: { maxTries: 1 }, + // Make sure socket is closed once the operation is done. + keepAliveOptions: { enable: false } + }) + ); + + const containerName: string = getUniqueName("1container-with-dash"); + const containerClient = serviceClient.getContainerClient(containerName); + await containerClient.create(); + + const content = "Hello World"; + const sourceClient = containerClient.getBlockBlobClient( + getUniqueName("source") + ); + await sourceClient.upload(content, content.length); + + const targetClient = containerClient.getBlockBlobClient( + getUniqueName("target") + ); + const blockId = Buffer.from("block").toString("base64"); + await targetClient.stageBlockFromURL( + blockId, + sourceClient.url, + 0, + undefined, + { sourceAuthorization: { scheme: "Bearer", value: token } } + ); + await targetClient.commitBlockList([blockId]); + + const downloaded = await targetClient.downloadToBuffer(); + assert.deepStrictEqual(downloaded.toString(), content); + + await containerClient.delete(); + }); + + it(`Should not authorize a copy source with an expired x-ms-copy-source-authorization token @loki @sql`, async () => { + const token = generateJWTToken( + new Date("2019/01/01"), + new Date("2019/01/01"), + new Date("2100/01/01"), + "https://sts.windows-ppe.net/ab1f708d-50f6-404c-a006-d71b2ac7a606/", + "https://storage.azure.com", + "user_impersonation", + "23657296-5cd5-45b0-a809-d972a7f4dfe1", + "dd0d0df1-06c3-436c-8034-4b9a153097ce" + ); + const expiredToken = generateJWTToken( + new Date("2019/01/01"), + new Date("2019/01/01"), + new Date("2019/01/01"), + "https://sts.windows-ppe.net/ab1f708d-50f6-404c-a006-d71b2ac7a606/", + "https://storage.azure.com", + "user_impersonation", + "23657296-5cd5-45b0-a809-d972a7f4dfe1", + "dd0d0df1-06c3-436c-8034-4b9a153097ce" + ); + + const serviceClient = new BlobServiceClient( + baseURL, + newPipeline(new SimpleTokenCredential(token), { + retryOptions: { maxTries: 1 }, + // Make sure socket is closed once the operation is done. + keepAliveOptions: { enable: false } + }) + ); + + const containerName: string = getUniqueName("1container-with-dash"); + const containerClient = serviceClient.getContainerClient(containerName); + await containerClient.create(); + + const content = "Hello World"; + const sourceClient = containerClient.getBlockBlobClient( + getUniqueName("source") + ); + await sourceClient.upload(content, content.length); + + const targetClient = containerClient.getBlockBlobClient( + getUniqueName("target") + ); + let error; + try { + await targetClient.syncUploadFromURL(sourceClient.url, { + sourceAuthorization: { scheme: "Bearer", value: expiredToken } + }); + } catch (err) { + error = err; + } finally { + await containerClient.delete(); + } + assert.ok(error, "Put Blob From URL should fail with an expired source token"); + assert.deepStrictEqual(error.code, "CannotVerifyCopySource"); + assert.deepStrictEqual(error.statusCode, 403); + }); + + it(`Should authorize a copy source by its SAS when x-ms-copy-source-authorization is also set @loki @sql`, async () => { + const token = generateJWTToken( + new Date("2019/01/01"), + new Date("2019/01/01"), + new Date("2100/01/01"), + "https://sts.windows-ppe.net/ab1f708d-50f6-404c-a006-d71b2ac7a606/", + "https://storage.azure.com", + "user_impersonation", + "23657296-5cd5-45b0-a809-d972a7f4dfe1", + "dd0d0df1-06c3-436c-8034-4b9a153097ce" + ); + + const serviceClient = new BlobServiceClient( + baseURL, + newPipeline(new SimpleTokenCredential(token), { + retryOptions: { maxTries: 1 }, + // Make sure socket is closed once the operation is done. + keepAliveOptions: { enable: false } + }) + ); + + const containerName: string = getUniqueName("1container-with-dash"); + const containerClient = serviceClient.getContainerClient(containerName); + await containerClient.create(); + + const content = "Hello World"; + const sourceBlobName = getUniqueName("source"); + const sourceClient = containerClient.getBlockBlobClient(sourceBlobName); + await sourceClient.upload(content, content.length); + + const expiresOn = new Date(); + expiresOn.setHours(expiresOn.getHours() + 1); + const sourceSAS = generateBlobSASQueryParameters( + { + containerName, + blobName: sourceBlobName, + expiresOn, + permissions: ContainerSASPermissions.parse("r") + }, + new StorageSharedKeyCredential( + EMULATOR_ACCOUNT_NAME, + EMULATOR_ACCOUNT_KEY + ) + ); + + const targetClient = containerClient.getBlockBlobClient( + getUniqueName("target") + ); + await targetClient.syncUploadFromURL(`${sourceClient.url}?${sourceSAS}`, { + sourceAuthorization: { scheme: "Bearer", value: "invalid" } + }); + + const downloaded = await targetClient.downloadToBuffer(); + assert.deepStrictEqual(downloaded.toString(), content); + + await containerClient.delete(); + }); + + it(`Should accept a lowercase bearer scheme in x-ms-copy-source-authorization @loki @sql`, async () => { + const token = generateJWTToken( + new Date("2019/01/01"), + new Date("2019/01/01"), + new Date("2100/01/01"), + "https://sts.windows-ppe.net/ab1f708d-50f6-404c-a006-d71b2ac7a606/", + "https://storage.azure.com", + "user_impersonation", + "23657296-5cd5-45b0-a809-d972a7f4dfe1", + "dd0d0df1-06c3-436c-8034-4b9a153097ce" + ); + + const serviceClient = new BlobServiceClient( + baseURL, + newPipeline(new SimpleTokenCredential(token), { + retryOptions: { maxTries: 1 }, + // Make sure socket is closed once the operation is done. + keepAliveOptions: { enable: false } + }) + ); + + const containerName: string = getUniqueName("1container-with-dash"); + const containerClient = serviceClient.getContainerClient(containerName); + await containerClient.create(); + + const content = "Hello World"; + const sourceClient = containerClient.getBlockBlobClient( + getUniqueName("source") + ); + await sourceClient.upload(content, content.length); + + const targetClient = containerClient.getBlockBlobClient( + getUniqueName("target") + ); + await targetClient.syncUploadFromURL(sourceClient.url, { + sourceAuthorization: { scheme: "bearer", value: token } + }); + + const downloaded = await targetClient.downloadToBuffer(); + assert.deepStrictEqual(downloaded.toString(), content); + + await containerClient.delete(); + }); + + for (const operation of ["Put Blob From URL", "Put Block From URL"]) { + it(`${operation} should authorize a private source with multiple bearer separator spaces @loki @sql`, async () => { + const token = generateJWTToken( + new Date("2019/01/01"), + new Date("2019/01/01"), + new Date("2100/01/01"), + "https://sts.windows-ppe.net/ab1f708d-50f6-404c-a006-d71b2ac7a606/", + "https://storage.azure.com", + "user_impersonation", + "23657296-5cd5-45b0-a809-d972a7f4dfe1", + "dd0d0df1-06c3-436c-8034-4b9a153097ce" + ); + const serviceClient = new BlobServiceClient( + baseURL, + newPipeline(new SimpleTokenCredential(token), { + retryOptions: { maxTries: 1 }, + keepAliveOptions: { enable: false } + }) + ); + const containerClient = serviceClient.getContainerClient( + getUniqueName("container") + ); + await containerClient.create(); + + try { + const content = "Hello World"; + const sourceClient = containerClient.getBlockBlobClient( + getUniqueName("source") + ); + await sourceClient.upload(content, content.length); + + for (const scheme of ["Bearer", "bearer"]) { + for (const separatorSpaces of [2, 3]) { + const targetClient = containerClient.getBlockBlobClient( + getUniqueName("target") + ); + const sourceAuthorization = { + scheme, + // The SDK already inserts the first separator space. + value: `${" ".repeat(separatorSpaces - 1)}${token}` + }; + if (operation === "Put Blob From URL") { + await targetClient.syncUploadFromURL(sourceClient.url, { + sourceAuthorization + }); + } else { + const blockId = Buffer.from("block").toString("base64"); + await targetClient.stageBlockFromURL( + blockId, + sourceClient.url, + 0, + content.length, + { sourceAuthorization } + ); + await targetClient.commitBlockList([blockId]); + } + assert.deepStrictEqual( + (await targetClient.downloadToBuffer()).toString(), + content, + `${operation}, ${scheme}, ${separatorSpaces} separator spaces` + ); + } + } + + const targetClient = containerClient.getBlockBlobClient( + getUniqueName("target") + ); + const sourceAuthorization = { scheme: "Bearer", value: " " }; + await assert.rejects( + operation === "Put Blob From URL" + ? targetClient.syncUploadFromURL(sourceClient.url, { + sourceAuthorization + }) + : targetClient.stageBlockFromURL( + Buffer.from("block").toString("base64"), + sourceClient.url, + 0, + content.length, + { sourceAuthorization } + ), + { code: "CannotVerifyCopySource", statusCode: 403 } + ); + } finally { + await containerClient.delete(); + } + }); + } + + it(`Should not authorize a copy source with a SharedKey x-ms-copy-source-authorization @loki @sql`, async () => { + const token = generateJWTToken( + new Date("2019/01/01"), + new Date("2019/01/01"), + new Date("2100/01/01"), + "https://sts.windows-ppe.net/ab1f708d-50f6-404c-a006-d71b2ac7a606/", + "https://storage.azure.com", + "user_impersonation", + "23657296-5cd5-45b0-a809-d972a7f4dfe1", + "dd0d0df1-06c3-436c-8034-4b9a153097ce" + ); + + const serviceClient = new BlobServiceClient( + baseURL, + newPipeline(new SimpleTokenCredential(token), { + retryOptions: { maxTries: 1 }, + // Make sure socket is closed once the operation is done. + keepAliveOptions: { enable: false } + }) + ); + + const containerName: string = getUniqueName("1container-with-dash"); + const containerClient = serviceClient.getContainerClient(containerName); + await containerClient.create(); + + const content = "Hello World"; + const sourceClient = containerClient.getBlockBlobClient( + getUniqueName("source") + ); + await sourceClient.upload(content, content.length); + + // A signature that the shared key authenticator accepts for the source + // read: a GET with no date or x-ms- headers, optionally with a range. + const sign = (range: string = "") => + computeHMACSHA256( + `GET\n\n\n\n\n\n\n\n\n\n\n${range}\n` + + `/${EMULATOR_ACCOUNT_NAME}${new URL(sourceClient.url).pathname}`, + Buffer.from(EMULATOR_ACCOUNT_KEY, "base64") + ); + const sourceRange = `bytes=0-${content.length - 1}`; + + const targetClient = containerClient.getBlockBlobClient( + getUniqueName("target") + ); + const blockId = Buffer.from("block").toString("base64"); + const errors: any[] = []; + try { + for (const copy of [ + () => + targetClient.syncUploadFromURL(sourceClient.url, { + sourceAuthorization: { + scheme: "SharedKey", + value: `${EMULATOR_ACCOUNT_NAME}:${sign()}` + } + }), + () => + targetClient.stageBlockFromURL( + blockId, + sourceClient.url, + 0, + content.length, + { + sourceAuthorization: { + scheme: "SharedKey", + value: `${EMULATOR_ACCOUNT_NAME}:${sign(sourceRange)}` + } + } + ) + ]) { + try { + await copy(); + errors.push(undefined); + } catch (err) { + errors.push(err); + } + } + } finally { + await containerClient.delete(); + } + + for (const [i, operation] of [ + "Put Blob From URL", + "Put Block From URL" + ].entries()) { + const error = errors[i]; + assert.ok(error, `${operation} should fail with a SharedKey source credential`); + assert.deepStrictEqual(error.code, "InvalidHeaderValue"); + assert.deepStrictEqual(error.statusCode, 400); + assert.ok(!error.message.includes(sign())); + assert.ok(!error.message.includes(sign(sourceRange))); + } + }); + it(`Should not work with HTTP @loki @sql`, async () => { await server.close(); await server.clean();