Add Create Multipart Data keyword for streamed multipart/form-data uploads

.github/workflows/pythonapp.yml

@@ -12,6 +12,13 @@ jobs:
       matrix:
         os: [ ubuntu-latest, macos-latest, windows-latest ]
         python-version: [ 3.8, 3.12 ]
+        # Python 3.8 is not available anymore for ubuntu-latest (Ubuntu 26.04)
+        exclude:
+          - os: ubuntu-latest
+            python-version: 3.8
+        include:
+          - os: ubuntu-24.04
+            python-version: 3.8
     steps:
       - uses: actions/checkout@v4
       - name: Set up Python

README.md

@@ -101,6 +101,36 @@ Post Request Test
     Status Should Be                 201  ${resp}                                                      
 ```
 
+### 📎 Multipart form data upload
+
+`Create Multipart Data` builds a `multipart/form-data` body, the format used by HTML forms and most
+APIs to upload files, to pass as `data` to `POST On Session`, `PUT On Session` or `PATCH On Session`.
+Files are streamed from disk, so large files are not loaded in memory.
+It needs an optional dependency: `pip install "robotframework-requests[multipart]"`
+
+- `fields`: dictionary of `name=value`, or list of `(name, value)` pairs to repeat a name.
+  Dictionary and list values are sent as JSON.
+- `files`: dictionary of `name=path`, or list of `(name, path[, content_type[, filename]])`,
+  `(name, [paths])` or `{name, path, content_type, filename}` dictionaries.
+  The file name and content type are taken from the path when not given.
+- `files_first=${True}` sends the files before the fields.
+- The `Content-Type` header with its boundary is set for you, session headers such as an
+  `Authorization` token are kept.
+
+```robotframework
+Upload Report Test
+    ${headers}=      Create Dictionary    Authorization=Bearer ${token}
+    Create Session   api    https://example.com    headers=${headers}
+    ${settings}=     Create Dictionary    everything=${True}
+    ${fields}=       Create Dictionary    action=add    settings=${settings}
+    ${files}=        Create Dictionary    document=${CURDIR}/report.pdf
+    ${multipart}=    Create Multipart Data    fields=${fields}    files=${files}
+    ${resp}=         POST On Session    api    /upload    data=${multipart}
+```
+
+See the [Create Multipart Data](http://marketsquare.github.io/robotframework-requests/doc/RequestsLibrary.html#Create%20Multipart%20Data)
+keyword documentation for all the formats and examples.
+
 ### 📖 Keywords documentation
 
 Robotframework-requests offers a wide set of keywords which can be found in

atests/test_post_multipart.robot

@@ -1,5 +1,6 @@
 *** Settings ***
 Library     RequestsLibrary
+Library     OperatingSystem
 
 
 *** Test Cases ***
@@ -42,3 +43,126 @@ Test Post List On Session Multipart
     Should Contain    ${resp.json()}[headers][Content-Length]    466
     Should Contain    ${resp.json()}[files]    randombytes
     Length Should Be    ${resp.json()}[files][randombytes]    2
+
+Test Post Multipart Data With Fields And Files On Session
+    ${settings}=    Create Dictionary    everything=${True}
+    ${fields}=    Create Dictionary    action=add    settings=${settings}
+    ${files}=    Create Dictionary    document=${CURDIR}${/}data.json
+    ${multipart}=    Create Multipart Data    fields=${fields}    files=${files}
+    ${file}=    Set Variable    ${multipart.fields}[2][1][1]
+
+    Should Not Be True    ${file.closed}
+
+    ${resp}=    POST On Session    ${GLOBAL_SESSION}    /anything    data=${multipart}
+
+    Should Be True    ${file.closed}
+    Should Be Equal    ${resp.json()}[headers][Content-Type]    ${multipart.content_type}
+    Should Be Equal    ${resp.json()}[form][action]    add
+    Should Be Equal    ${resp.json()}[form][settings]    {"everything": true}
+    # raw bytes: Get File would turn the CRLF line endings of a Windows checkout into LF
+    ${expected}=    Get Binary File    ${CURDIR}${/}data.json
+    Should Be Equal    ${resp.json()}[files][document]    ${expected.decode('utf-8')}
+
+Test Put Multipart Data With Repeated Fields And Files On Session
+    ${tag_1}=    Create List    tag    one
+    ${tag_2}=    Create List    tag    two
+    ${fields}=    Create List    ${tag_1}    ${tag_2}
+    ${file_1}=    Create List    attachments    ${CURDIR}${/}data.json
+    ${file_2}=    Create List    attachments    ${CURDIR}${/}index.html    text/html    renamed.html
+    ${files}=    Create List    ${file_1}    ${file_2}
+    ${multipart}=    Create Multipart Data    fields=${fields}    files=${files}
+
+    ${resp}=    PUT On Session    ${GLOBAL_SESSION}    /anything    data=${multipart}
+
+    Should Contain    ${resp.json()}[headers][Content-Type]    multipart/form-data; boundary=
+    Should Be Equal    ${resp.json()}[form][tag]    ${{['one', 'two']}}
+    Length Should Be    ${resp.json()}[files][attachments]    2
+
+Test Post Large Multipart Data With Fields
+    ${large_file}=    Set Variable    ${OUTPUT_DIR}${/}large_multipart.bin
+    Create Binary File    ${large_file}    ${{os.urandom(50000)}}
+    ${fields}=    Create Dictionary    name=large
+    ${files}=    Create Dictionary    file=${large_file}
+    ${multipart}=    Create Multipart Data    fields=${fields}    files=${files}
+
+    ${resp}=    POST    ${HTTP_LOCAL_SERVER}/anything    data=${multipart}
+
+    Should Be Equal    ${resp.json()}[form][name]    large
+    Should Be True    int($resp.json()['headers']['Content-Length']) > 50000
+    [Teardown]    Remove File    ${large_file}
+
+Test Post Multipart Data Keeps Explicit Content Type
+    ${files}=    Create Dictionary    document=${CURDIR}${/}data.json
+    ${multipart}=    Create Multipart Data    files=${files}
+    ${headers}=    Create Dictionary    Content-Type=${multipart.content_type}; charset=utf-8
+
+    ${resp}=    POST On Session    ${GLOBAL_SESSION}    /anything    data=${multipart}    headers=${headers}
+
+    Should Be Equal    ${resp.json()}[headers][Content-Type]    ${multipart.content_type}; charset=utf-8
+
+Test Post Multipart Data With Files First And A List Of Paths
+    ${fields}=    Create Dictionary    action=add
+    ${paths}=    Create List    ${CURDIR}${/}data.json    ${CURDIR}${/}index.html
+    ${attachments}=    Create List    attachments    ${paths}
+    ${files}=    Create List    ${attachments}
+    ${multipart}=    Create Multipart Data    fields=${fields}    files=${files}    files_first=${True}
+
+    Should Be Equal    ${multipart.fields}[0][0]    attachments
+    Should Be Equal    ${multipart.fields}[1][0]    attachments
+    Should Be Equal    ${multipart.fields}[2][0]    action
+
+    ${resp}=    POST On Session    ${GLOBAL_SESSION}    /anything    data=${multipart}
+
+    Should Be Equal    ${resp.json()}[form][action]    add
+    Length Should Be    ${resp.json()}[files][attachments]    2
+
+Test Post Multipart Data Keeps Session Headers
+    ${session_headers}=    Create Dictionary    Authorization=Bearer some-token    Content-Type=application/json
+    Create Session    bearer_session    ${HTTP_LOCAL_SERVER}    headers=${session_headers}
+    ${files}=    Create Dictionary    document=${CURDIR}${/}data.json
+    ${multipart}=    Create Multipart Data    files=${files}
+
+    ${resp}=    POST On Session    bearer_session    /anything    data=${multipart}
+
+    Should Be Equal    ${resp.json()}[headers][Authorization]    Bearer some-token
+    Should Be Equal    ${resp.json()}[headers][Content-Type]    ${multipart.content_type}
+
+Test Post Multipart Data Is Sent Whole Again After A 307 Redirect
+    ${fields}=    Create Dictionary    action=add
+    ${files}=    Create Dictionary    document=${CURDIR}${/}data.json
+    ${multipart}=    Create Multipart Data    fields=${fields}    files=${files}
+
+    ${resp}=    POST On Session    ${GLOBAL_SESSION}    /redirect-to    params=url=/anything&status_code=307    data=${multipart}
+
+    Should Be Equal As Strings    ${resp.history[0].status_code}    307
+    Should Be Equal    ${resp.json()}[form][action]    add
+    ${expected}=    Get Binary File    ${CURDIR}${/}data.json
+    Should Be Equal    ${resp.json()}[files][document]    ${expected.decode('utf-8')}
+
+Test Post Multipart Data With Files Described By Dictionaries
+    ${file}=    Create Dictionary    name=document    path=${CURDIR}${/}data.json
+    ...    filename=renamed.json    content_type=application/json
+    ${files}=    Create List    ${file}
+    ${fields}=    Create Dictionary    count=${3}
+    ${multipart}=    Create Multipart Data    fields=${fields}    files=${files}
+
+    Should Be Equal    ${multipart.fields}[1][1][0]    renamed.json
+    Should Be Equal    ${multipart.fields}[1][1][2]    application/json
+
+    ${resp}=    POST On Session    ${GLOBAL_SESSION}    /anything    data=${multipart}
+
+    Should Be Equal    ${resp.json()}[form][count]    3
+    ${expected}=    Get Binary File    ${CURDIR}${/}data.json
+    Should Be Equal    ${resp.json()}[files][document]    ${expected.decode('utf-8')}
+
+Test Create Multipart Data Fails Clearly With Invalid Arguments
+    Run Keyword And Expect Error    ValueError: Multipart fields must be a dictionary or a list of (name, value) pairs*
+    ...    Create Multipart Data    fields=${{["not a pair"]}}
+    Run Keyword And Expect Error    ValueError: Multipart files must be (name, path*
+    ...    Create Multipart Data    files=${{[("document",)]}}
+    Run Keyword And Expect Error    ValueError: Multipart file needs a name and a path*
+    ...    Create Multipart Data    files=${{[{"name": "document"}]}}
+    ${existing}=    Create List    document    ${CURDIR}${/}data.json
+    ${missing}=    Create List    missing    ${CURDIR}${/}does_not_exist.bin
+    Run Keyword And Expect Error    FileNotFoundError: *does_not_exist.bin*
+    ...    Create Multipart Data    files=${{[$existing, $missing]}}

setup.py

@@ -21,7 +21,10 @@
 Topic :: Software Development :: Testing
 """[1:-1]
 
-TEST_REQUIRE = ['robotframework>=3.2.1', 'pytest', 'flask', 'six', 'coverage', 'flake8', 'Flask-HTTPAuth==4.8.0']
+MULTIPART_REQUIRE = ['requests-toolbelt']
+
+TEST_REQUIRE = ['robotframework>=3.2.1', 'pytest', 'flask', 'six', 'coverage', 'flake8', 'Flask-HTTPAuth==4.8.0'] \
+    + MULTIPART_REQUIRE
 
 VERSION = None
 version_file = join(dirname(abspath(__file__)), 'src', 'RequestsLibrary', 'version.py')
@@ -53,5 +56,6 @@
           'requests'
       ],
       extras_require={
+          'multipart': MULTIPART_REQUIRE,
           'test': TEST_REQUIRE
       })

src/RequestsLibrary/RequestsKeywords.py

@@ -1,13 +1,17 @@
 import requests
 import robot
+from requests.structures import CaseInsensitiveDict
 from robot.api.deco import keyword
 from robot.libraries.BuiltIn import BuiltIn
 
 from RequestsLibrary import log
 from RequestsLibrary.compat import urljoin
+from RequestsLibrary.multipart import create_multipart_encoder
 from RequestsLibrary.utils import (
     is_list_or_tuple,
     is_file_descriptor,
+    is_multipart_encoder,
+    get_multipart_file_descriptors,
     warn_if_equal_symbol_in_url_session_less,
 )
 
@@ -32,6 +36,13 @@ def _common_request(self, method, session, uri, **kwargs):
 
         self._capture_output()
 
+        data = kwargs.get("data")
+        if is_multipart_encoder(data):
+            # The boundary is generated by the encoder, an explicit Content-Type still wins
+            headers = CaseInsensitiveDict(kwargs.get("headers") or {})
+            headers.setdefault("Content-Type", data.content_type)
+            kwargs["headers"] = headers
+
         resp = request_function(
             method,
             self._merge_url(session, uri),
@@ -68,7 +79,10 @@ def _close_file_descriptors(files, data):
             files_descriptor_to_close = filter(
                 is_file_descriptor, list(files.values()) + [data]
             )
-
+
+        if is_multipart_encoder(data):
+            files_descriptor_to_close = list(files_descriptor_to_close) + get_multipart_file_descriptors(data)
+
         for file_descriptor in files_descriptor_to_close:
             file_descriptor.close()
 
@@ -168,6 +182,98 @@ def get_file_for_streaming_upload(path):
         """
         return open(path, "rb")
 
+    @staticmethod
+    @keyword("Create Multipart Data")
+    def create_multipart_data(fields=None, files=None, files_first=False):
+        """
+        Creates a ``multipart/form-data`` body, made of form ``fields`` and ``files``, the format used
+        by HTML forms and by most APIs to upload files.
+
+        The returned body is passed as ``data`` to `POST On Session`, `PUT On Session`, `PATCH On Session`
+        or to the session-less `POST`, `PUT` and `PATCH` keywords. Files are streamed from disk while the
+        request is sent instead of being read into memory, so it is suited for large uploads too.
+
+        This keyword requires the ``requests-toolbelt`` package, it can be installed with
+        ``pip install "robotframework-requests[multipart]"``.
+
+        *Form fields*
+
+        ``fields`` is a dictionary of ``name=value`` or a list of ``(name, value)`` pairs.
+        A list allows to send the same field name several times.
+        Values that are dictionaries or lists are sent as JSON strings, other values as strings.
+
+        | ${settings}=    Create Dictionary    everything=${True}
+        | ${fields}=      Create Dictionary    action=add    settings=${settings}
+        | # sent as: action = add, settings = {"everything": true}
+        |
+        | ${tag_1}=       Create List    tag    one
+        | ${tag_2}=       Create List    tag    two
+        | ${fields}=      Create List    ${tag_1}    ${tag_2}
+        | # sent as: tag = one, tag = two
+
+        *Files*
+
+        ``files`` is a dictionary of ``name=path``, or a list where each file is described with one of
+        the following formats. A list allows to send several files with the same name.
+
+        | = Format =                               | = Description =                                                     |
+        | ``(name, path)``                         | File name and content type are taken from ``path``.                 |
+        | ``(name, path, content_type)``           | With its own content type.                                          |
+        | ``(name, path, content_type, filename)`` | With its own file name, ``content_type`` can be ``${None}``.        |
+        | ``(name, [path, ...])``                  | Several files with the same name.                                   |
+        | ``{name, path, content_type, filename}`` | A dictionary, ``content_type`` and ``filename`` keys are optional.  |
+
+        | ${file_1}=    Create List    document    ${CURDIR}/report.pdf
+        | ${file_2}=    Create List    document    ${CURDIR}/report    application/pdf
+        | ${file_3}=    Create List    document    ${CURDIR}/r.pdf    ${None}    report.pdf
+        | ${file_4}=    Create List    attachments    ${paths}
+        | ${file_5}=    Create Dictionary    name=document    path=${CURDIR}/report.pdf
+        | ${files}=     Create List    ${file_1}    ${file_4}
+
+        In a dictionary of ``name=path``, the path can also be a list of paths: ``attachments=${paths}``.
+        ``filename``, the file name given to the server, defaults to the base name of ``path``.
+        ``content_type`` defaults to the type guessed from the file name extension,
+        or ``application/octet-stream`` when it is unknown.
+
+        *Order of the parts*
+
+        Parts are sent in the given order, fields first and then files.
+        Set ``files_first`` to ``${True}`` for servers that expect the files before the fields.
+
+        *Sending the body*
+
+        The ``Content-Type`` header, ``multipart/form-data`` with the generated boundary, is set
+        automatically when the request is sent, unless a ``Content-Type`` is explicitly passed in the
+        request ``headers``. Session headers, like an ``Authorization`` token, are sent as usual and
+        a ``Content-Type`` set on the session is replaced for this request only.
+
+        Files are opened by this keyword and closed by the requests keyword once the request is sent,
+        so create a new body for each request. Within that request the body can be sent again from its
+        start, so session retries and redirects resend it whole.
+
+        *Examples*
+
+        A file with form fields, on a session authenticated with a Bearer token:
+        | ${headers}=      Create Dictionary    Authorization=Bearer ${token}
+        | Create Session   api    https://example.com    headers=${headers}
+        | ${settings}=     Create Dictionary    everything=${True}
+        | ${fields}=       Create Dictionary    action=add    settings=${settings}
+        | ${files}=        Create Dictionary    document=${CURDIR}/report.pdf
+        | ${multipart}=    Create Multipart Data    fields=${fields}    files=${files}
+        | ${resp}=         POST On Session    api    /upload    data=${multipart}
+
+        Several files with the same name, a file with its own content type and file name,
+        sent before the fields:
+        | ${paths}=        Create List    ${CURDIR}/a.txt    ${CURDIR}/b.txt
+        | ${attachments}=  Create List    attachments    ${paths}
+        | ${metadata}=     Create List    metadata    ${CURDIR}/meta    application/xml    meta.xml
+        | ${files}=        Create List    ${attachments}    ${metadata}
+        | ${fields}=       Create Dictionary    action=add
+        | ${multipart}=    Create Multipart Data    fields=${fields}    files=${files}    files_first=${True}
+        | ${resp}=         PUT On Session    api    /upload    data=${multipart}
+        """
+        return create_multipart_encoder(fields, files, files_first)
+
     @keyword("Last response")
     def get_last_response(self) -> requests.Response:
         """
@@ -195,7 +301,7 @@ def session_less_get(
 
         Other optional requests arguments can be passed using ``**kwargs`` here is a list:
 
-        | ``data``     | Dictionary, list of tuples, bytes, or file-like object to send in the body of the request. |
+        | ``data``     | Dictionary, list of tuples, bytes, or file-like object to send in the body of the request. It can also be a multipart body created with `Create Multipart Data`. |
         | ``json``     | A JSON serializable Python object to send in the body of the request. |
         | ``headers``  | Dictionary of HTTP Headers to send with the request. |
         | ``cookies``  | Dict or CookieJar object to send with the request. |

src/RequestsLibrary/log.py

@@ -42,7 +42,9 @@ def format_data_to_log_string(data, limit=LOG_CHAR_LIMIT):
     if not data:
         return None
 
-    if is_file_descriptor(data):
+    # Streaming bodies (file descriptors, generators, multipart encoders...)
+    # can't be read without consuming them and may not support len() or slicing
+    if is_file_descriptor(data) or not isinstance(data, (str, bytes)):
         return repr(data)
 
     if logging.getLogger().level > logging.DEBUG and len(data) > limit: