Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
Deprecated
----------

- ``FileTokenStorage.user_only_umask()`` and ``FileAdapter.user_only_umask()``
are deprecated and will be removed in version 5. (:pr:`NUMBER`)

They change the umask of the whole process,
which is not safe when other threads are running.

Fixed
-----

- All file-based token storage classes create files
with mode ``0o600`` set directly, and do not change the process umask. (:pr:`NUMBER`)

This avoids a thread safety issue that could cause I/O in other threads
to create files with an unexpected umask.
47 changes: 47 additions & 0 deletions src/globus_sdk/token_storage/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -200,6 +200,37 @@ def file_exists(self) -> bool:
"""
return os.path.exists(self.filepath)

@staticmethod
def _owner_only_opener(path: str, flags: int) -> int:
"""
An ``opener`` for the builtin ``open()``.

A new file will be created with the mode ``0o600``.
An existing file will keep its existing mode.
"""

try:
# Open the file with a 0o600 mode set. (See below for a corner case.)
# The O_CREAT | O_EXCL flags will trigger `FileExistsError`
# if the file already exists.
fd = os.open(path, flags | os.O_CREAT | os.O_EXCL, 0o600)
except FileExistsError:
# The file already exists. Open and return the descriptor immediately
# to avoid falling through to the code below that changes the file mode.
# Note: a dangling symlink will reach this line and raise an exception.
return os.open(path, flags & ~os.O_CREAT)

# Corner case: the 0o600 mode, set above, may be further restricted
# by the process-level umask. Therefore, set the permissions exactly.
try:
os.fchmod(fd, 0o600)
except (AttributeError, OSError):
# `os.fchmod` doesn't exist on all platforms.
# In addition, some filesystems do not support changing modes.
pass

return fd

@contextlib.contextmanager
def user_only_umask(self) -> t.Iterator[None]:
"""
Expand All @@ -213,7 +244,23 @@ def user_only_umask(self) -> t.Iterator[None]:
On Windows, this has no effect. To control the permissions on files used for
token storage, use ``%LOCALAPPDATA%`` or ``%APPDATA%``.
These directories should only be accessible to the current user.

.. warning::

This method is deprecated and will be removed in version 5.

The umask of the entire Python process is changed, so it is not safe
to use this method while other threads create files or use this method.
"""

# In a context manager, the frames above this one are contextlib's
# ``__enter__`` and then the caller's ``with`` statement.
globus_sdk.exc.warn_deprecated(
"'user_only_umask' is deprecated and will be removed in version 5. "
"It changes the umask of the whole process, "
"which is not safe when other threads are running.",
stacklevel=4,
)
old_umask = os.umask(0o177)
try:
yield
Expand Down
21 changes: 11 additions & 10 deletions src/globus_sdk/token_storage/json.py
Original file line number Diff line number Diff line change
Expand Up @@ -130,8 +130,7 @@ def store_token_data_by_resource_server(
being written to the file at ``self.filepath``.

Under the assumption that this may be running on a system with multiple
local users, this sets the umask such that only the owner of the
resulting file can read or write it.
local users, a newly created file can be read and written only by its owner.

:param token_data_by_resource_server: A mapping of resource servers to token
data.
Expand All @@ -149,10 +148,11 @@ def store_token_data_by_resource_server(
# update globus-sdk version
to_write["globus-sdk.version"] = __version__

# write the file, denying rwx to Group and World, exec to User
with self.user_only_umask():
with open(self.filepath, "w", encoding="utf-8") as f:
json.dump(to_write, f)
# write the file (a newly created file is owner read/write only)
with open(
self.filepath, "w", encoding="utf-8", opener=self._owner_only_opener
) as f:
json.dump(to_write, f)

def get_token_data_by_resource_server(self) -> dict[str, TokenStorageData]:
"""
Expand Down Expand Up @@ -180,9 +180,10 @@ def remove_token_data(self, resource_server: str) -> bool:
# pop the token data out if it exists
popped = to_write["data"].get(self.namespace, {}).pop(resource_server, None)

# overwrite the file, denying rwx to Group and World, exec to User
with self.user_only_umask():
with open(self.filepath, "w", encoding="utf-8") as f:
json.dump(to_write, f)
# overwrite the file (a newly created file is owner read/write only)
with open(
self.filepath, "w", encoding="utf-8", opener=self._owner_only_opener
) as f:
json.dump(to_write, f)

return popped is not None
47 changes: 47 additions & 0 deletions src/globus_sdk/token_storage/legacy/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,37 @@ def file_exists(self) -> bool:
"""
return os.path.exists(self.filename)

@staticmethod
def _owner_only_opener(path: str, flags: int) -> int:
"""
An ``opener`` for the builtin ``open()``.

A new file will be created with the mode ``0o600``.
An existing file will keep its existing mode.
"""

try:
# Open the file with a 0o600 mode set. (See below for a corner case.)
# The O_CREAT | O_EXCL flags will trigger `FileExistsError`
# if the file already exists.
fd = os.open(path, flags | os.O_CREAT | os.O_EXCL, 0o600)
except FileExistsError:
# The file already exists. Open and return the descriptor immediately
# to avoid falling through to the code below that changes the file mode.
# Note: a dangling symlink will reach this line and raise an exception.
return os.open(path, flags & ~os.O_CREAT)

# Corner case: the 0o600 mode, set above, may be further restricted
# by the process-level umask. Therefore, set the permissions exactly.
try:
os.fchmod(fd, 0o600)
except (AttributeError, OSError):
# `os.fchmod` doesn't exist on all platforms.
# In addition, some filesystems do not support changing modes.
pass

return fd

@contextlib.contextmanager
def user_only_umask(self) -> t.Iterator[None]:
"""
Expand All @@ -68,7 +99,23 @@ def user_only_umask(self) -> t.Iterator[None]:
On Windows, this has no effect. To control the permissions on files used for
token storage, use ``%LOCALAPPDATA%`` or ``%APPDATA%``.
These directories should only be accessible to the current user.

.. warning::

This method is deprecated and will be removed in version 5.

The umask of the entire Python process is changed, so it is not safe
to use this method while other threads create files or use this method.
"""

# In a context manager, the frames above this one are contextlib's
# ``__enter__`` and then the caller's ``with`` statement.
globus_sdk.exc.warn_deprecated(
"'user_only_umask' is deprecated and will be removed in version 5. "
"It changes the umask of the whole process, "
"which is not safe when other threads are running.",
stacklevel=4,
)
old_umask = os.umask(0o177)
try:
yield
Expand Down
12 changes: 6 additions & 6 deletions src/globus_sdk/token_storage/legacy/file_adapters.py
Original file line number Diff line number Diff line change
Expand Up @@ -114,8 +114,7 @@ def store(self, token_response: globus_sdk.OAuthTokenResponse) -> None:
which was in use.

Under the assumption that this may be running on a system with multiple
local users, this sets the umask such that only the owner of the
resulting file can read or write it.
local users, a newly created file can be read and written only by its owner.

:param token_response: The token data received from the refresh
"""
Expand All @@ -130,10 +129,11 @@ def store(self, token_response: globus_sdk.OAuthTokenResponse) -> None:
# newly received tokens
to_write["by_rs"].update(token_response.by_resource_server)

# deny rwx to Group and World, exec to User
with self.user_only_umask():
with open(self.filename, "w", encoding="utf-8") as f:
json.dump(to_write, f)
# Newly-created files have owner-only read/write permissions.
with open(
self.filename, "w", encoding="utf-8", opener=self._owner_only_opener
) as f:
json.dump(to_write, f)

def get_by_resource_server(self) -> dict[str, t.Any]:
"""
Expand Down
11 changes: 5 additions & 6 deletions src/globus_sdk/token_storage/legacy/sqlite_adapter.py
Original file line number Diff line number Diff line change
Expand Up @@ -61,12 +61,11 @@ def _init_and_connect(
init_tables = self._is_memory_db() or not self.file_exists()
connect_params = connect_params or {}
if init_tables and not self._is_memory_db(): # real file needs to be created
with self.user_only_umask():
conn: sqlite3.Connection = sqlite3.connect(
self.dbname, **connect_params
)
else:
conn = sqlite3.connect(self.dbname, **connect_params)
# Create the file with owner-only permissions before SQLite opens it.
# "Append" mode prevents an existing file from being truncated.
with open(self.dbname, "ab", opener=self._owner_only_opener):
pass
conn: sqlite3.Connection = sqlite3.connect(self.dbname, **connect_params)
Comment thread
kurtmckee marked this conversation as resolved.
Resolved
if init_tables:
conn.executescript("""
CREATE TABLE config_storage (
Expand Down
9 changes: 5 additions & 4 deletions src/globus_sdk/token_storage/sqlite.py
Original file line number Diff line number Diff line change
Expand Up @@ -51,10 +51,11 @@ def _init_and_connect(
) -> sqlite3.Connection:
connect_params = connect_params or {}
if not self.file_exists():
with self.user_only_umask():
conn: sqlite3.Connection = sqlite3.connect(
self.filepath, **connect_params
)
# Create the file with owner-only permissions before SQLite opens it.
# "Append" mode prevents an existing file from being truncated.
with open(self.filepath, "ab", opener=self._owner_only_opener):
pass
conn: sqlite3.Connection = sqlite3.connect(self.filepath, **connect_params)
conn.executescript(textwrap.dedent("""
CREATE TABLE token_storage (
namespace VARCHAR NOT NULL,
Expand Down
151 changes: 151 additions & 0 deletions tests/common/overlapping_writes.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,151 @@
"""
Helpers for tests which need two writers to overlap in one precise order,
such as tests of how writes interact with process-wide state (like the umask).

The order is enforced with events rather than sleeps, so it is the same on every run,
and it does not depend on how the code under test is implemented.
"""

from __future__ import annotations

import contextlib
import os
import sqlite3
import threading
import typing as t


def current_umask() -> int:
"""Read the process umask. (`os.umask` can only be read by setting it.)"""
current = os.umask(0)
os.umask(current)
return current


@contextlib.contextmanager
def process_umask(value: int) -> t.Iterator[None]:
"""Set the process umask for the duration of a block, then restore it."""
original = os.umask(value)
try:
yield
finally:
os.umask(original)


class OverlappingWrites:
"""
Run two writes in separate threads, in this order::

first starts -> second starts -> first finishes -> second finishes

Each write must call its checkpoint (``first_checkpoint`` or ``second_checkpoint``)
at a point where the code under test is mid-write.
The first write waits at its checkpoint until the second is also at its checkpoint.
The second write waits at its checkpoint until the first has completely finished.

If the code under test serializes the writes, the second can never reach its
checkpoint while the first waits, so the first simply times out and carries on.
The order of events is then different, but nothing deadlocks.

Usage
-----

An ``OverlappingWrites`` object is used once, and its lifecycle wraps
around the code under test: build the writes with its checkpoints,
then hand them to ``run()``, which starts and orders the threads.

.. code-block:: python

controller = OverlappingWrites()
action1 = build_action1(callback=controller.first_checkpoint)
action2 = build_action2(callback=controller.second_checkpoint)
controller.run(action1, action2)

# now assert on the state left behind by the overlapping writes

``run()`` waits for both threads, raises the first exception that either raised,
and fails if either write never reached its checkpoint.
"""

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can we add a little console block here to show demo usage? I really like this class, but it took a few minutes to read through usage sites and understand it.

We don't have to work with a real use-case, just something like...

# rephrase further as you see fit
controller = OverlappingWrites()
action1 = build_action1(callback=controller.first_checkpoint)
action2 = build_action2(callback=controller.second_checkpoint)
controller.run(action1, action2)

So that it's clear that the OverlappingWrites object lifecycle wraps around the code under test.


def __init__(self, timeout: float = 0.2) -> None:
self._timeout = timeout
self._first_started = threading.Event()
self._second_started = threading.Event()
self._first_finished = threading.Event()

def first_checkpoint(self) -> None:
self._first_started.set()
self._second_started.wait(self._timeout)

def second_checkpoint(self) -> None:
self._second_started.set()
self._first_finished.wait(self._timeout)

def run(self, first: t.Callable[[], t.Any], second: t.Callable[[], t.Any]) -> None:
errors: list[Exception] = []

def run_first() -> None:
try:
first()
except Exception as e:
errors.append(e)
finally:
self._first_finished.set()

def run_second() -> None:
try:
second()
except Exception as e:
errors.append(e)

first_thread = threading.Thread(target=run_first)
second_thread = threading.Thread(target=run_second)

first_thread.start()
assert self._first_started.wait(
5
), "the first write never reached its checkpoint"
second_thread.start()
first_thread.join(5)
second_thread.join(5)

assert not (first_thread.is_alive() or second_thread.is_alive())
assert (
self._second_started.is_set()
), "the second write never reached its checkpoint"
if errors:
raise errors[0]


class CheckpointDict(dict[str, t.Any]):
"""
A dict which calls a checkpoint when it is iterated.

When this is stored as token data, ``json.dump`` iterates it while writing, which
is after the file has been opened.
"""

def __init__(
self, checkpoint: t.Callable[[], None], data: dict[str, t.Any]
) -> None:
super().__init__(data)
self._checkpoint = checkpoint

def items(self) -> t.Any:
self._checkpoint()
return super().items()


def sqlite_factory_with_checkpoint(
checkpoint: t.Callable[[], None],
) -> t.Callable[..., sqlite3.Connection]:
"""
A ``factory`` for ``sqlite3.connect()`` which calls a checkpoint just before the
database file is opened (and therefore just before it is created).
"""

def factory(*args: t.Any, **kwargs: t.Any) -> sqlite3.Connection:
checkpoint()
return sqlite3.Connection(*args, **kwargs)

return factory
Loading
Loading