diff --git a/src/libvcs/_internal/run.py b/src/libvcs/_internal/run.py
index 6674e7e3..146f5b25 100644
--- a/src/libvcs/_internal/run.py
+++ b/src/libvcs/_internal/run.py
@@ -22,6 +22,18 @@
logger = logging.getLogger(__name__)
+console_encoding = sys.stdout.encoding
+
+
+def console_to_str(s: bytes) -> str:
+ """From pypa/pip project, pip.backwardwardcompat. License MIT."""
+ try:
+ return s.decode(console_encoding)
+ except UnicodeDecodeError:
+ return s.decode("utf_8")
+ except AttributeError: # for tests, #13
+ return str(s)
+
if t.TYPE_CHECKING:
_LoggerAdapter = logging.LoggerAdapter[logging.Logger]
@@ -78,7 +90,7 @@ def process(
class ProgressCallbackProtocol(t.Protocol):
"""Callback to report subprocess communication."""
- def __call__(self, output: t.AnyStr, timestamp: datetime.datetime) -> None:
+ def __call__(self, output: str, timestamp: datetime.datetime) -> None:
"""Process progress for subprocess communication."""
...
@@ -182,7 +194,7 @@ def progress_cb(output, timestamp):
restore_signals=restore_signals,
start_new_session=start_new_session,
pass_fds=pass_fds,
- text=True,
+ text=False, # Keep in bytes mode to preserve \r properly
encoding=encoding,
errors=errors,
user=user,
@@ -201,29 +213,36 @@ def progress_cb(output: t.AnyStr, timestamp: datetime.datetime) -> None:
sys.stdout.flush()
callback = progress_cb
+
+ # Note: When git detects that stderr is not a TTY (e.g., when piped),
+ # it outputs progress with newlines instead of carriage returns.
+ # This causes each progress update to appear on a new line.
+ # To get proper single-line progress updates, git would need to be
+ # connected to a pseudo-TTY, which would require significant changes
+ # to how subprocess execution is handled.
+
while code is None:
code = proc.poll()
if callback and callable(callback) and proc.stderr is not None:
- line = str(proc.stderr.read(128))
+ line = console_to_str(proc.stderr.read(128))
if line:
callback(output=line, timestamp=datetime.datetime.now())
if callback and callable(callback):
callback(output="\r", timestamp=datetime.datetime.now())
- lines = (
- filter(None, (line.strip() for line in proc.stdout.readlines()))
- if proc.stdout is not None
- else []
- )
- all_output = "\n".join(lines)
- if code:
- stderr_lines = (
- filter(None, (line.strip() for line in proc.stderr.readlines()))
- if proc.stderr is not None
- else []
+ if proc.stdout is not None:
+ lines: t.Iterable[bytes] = filter(
+ None, (line.strip() for line in proc.stdout.readlines())
+ )
+ all_output = console_to_str(b"\n".join(lines))
+ else:
+ all_output = ""
+ if code and proc.stderr is not None:
+ stderr_lines: t.Iterable[bytes] = filter(
+ None, (line.strip() for line in proc.stderr.readlines())
)
- all_output = "".join(stderr_lines)
+ all_output = console_to_str(b"".join(stderr_lines))
output = "".join(all_output)
if code != 0 and check_returncode:
raise exc.CommandError(
diff --git a/tests/_internal/test_run.py b/tests/_internal/test_run.py
new file mode 100644
index 00000000..f6df8965
--- /dev/null
+++ b/tests/_internal/test_run.py
@@ -0,0 +1,1491 @@
+"""Tests for libvcs._internal.run module, focusing on output streaming."""
+
+from __future__ import annotations
+
+import contextlib
+import sys
+import textwrap
+import time
+import typing as t
+
+import pytest
+
+from libvcs import exc
+from libvcs._internal.run import run
+
+if t.TYPE_CHECKING:
+ import datetime
+ import pathlib
+
+# Type alias for cleaner callback signatures
+CallbackOutput = str
+
+
+class TestProgressOutput:
+ """Test progress output streaming and flushing behavior."""
+
+ def test_progress_callback_receives_chunks(self, tmp_path: pathlib.Path) -> None:
+ """Test that progress callback receives output in chunks."""
+ captured_chunks: list[str] = []
+ captured_times: list[datetime.datetime] = []
+
+ def capture_callback(output: t.AnyStr, timestamp: datetime.datetime) -> None:
+ """Capture output chunks and timestamps."""
+ captured_chunks.append(str(output))
+ captured_times.append(timestamp)
+
+ # Create a test script that outputs progress
+ script = tmp_path / "progress_test.py"
+ script.write_text(
+ textwrap.dedent(
+ """
+ import sys
+ import time
+
+ # Simulate progress output to stderr
+ messages = [
+ "Starting process...\\n",
+ "Progress: 10%\\r",
+ "Progress: 20%\\r",
+ "Progress: 30%\\r",
+ "Progress: 40%\\r",
+ "Progress: 50%\\r",
+ "Done!\\n"
+ ]
+
+ for msg in messages:
+ sys.stderr.write(msg)
+ sys.stderr.flush()
+ time.sleep(0.01) # Small delay to ensure chunks
+ """,
+ ),
+ )
+
+ # Run the script with progress callback
+ run(
+ [sys.executable, str(script)],
+ log_in_real_time=True,
+ callback=capture_callback,
+ )
+
+ # Verify we got multiple chunks
+ assert len(captured_chunks) > 0, "Should receive progress chunks"
+ assert len(captured_times) == len(captured_chunks), (
+ "Each chunk should have timestamp"
+ )
+
+ # Check that final carriage return was sent
+ assert captured_chunks[-1] == "\r", "Should end with carriage return"
+
+ def test_no_fragmentation_without_newlines(self, tmp_path: pathlib.Path) -> None:
+ """Test that output without newlines is sent as one chunk after process ends."""
+ captured_chunks: list[str] = []
+
+ def capture_callback(output: t.AnyStr, timestamp: datetime.datetime) -> None:
+ """Capture output chunks."""
+ output_str = str(output)
+ if output_str != "\r": # Ignore the final \r
+ captured_chunks.append(output_str)
+
+ # Create a script that outputs a long line without newline
+ script = tmp_path / "long_line_test.py"
+ long_message = "X" * 300 # 300 characters without newline
+ script.write_text(
+ f"""
+import sys
+sys.stderr.write("{long_message}")
+sys.stderr.flush()
+""",
+ )
+
+ # Run with progress callback
+ run(
+ [sys.executable, str(script)],
+ log_in_real_time=True,
+ callback=capture_callback,
+ )
+
+ # With line-buffered reading, output without newlines
+ # should come as a single chunk after the process ends
+ assert len(captured_chunks) == 1, "Output without newline should be one chunk"
+
+ # Verify total output is preserved
+ total_output = "".join(captured_chunks)
+ assert total_output == long_message, "Total output should match original"
+
+ def test_git_style_progress_fragmentation(self, tmp_path: pathlib.Path) -> None:
+ """Test fragmentation of git-style progress output."""
+ captured_chunks: list[str] = []
+
+ def capture_callback(output: str, timestamp: datetime.datetime) -> None:
+ """Capture output chunks."""
+ if output != "\r": # Ignore the final \r
+ captured_chunks.append(output)
+
+ # Create a script that simulates git progress output
+ script = tmp_path / "git_progress_test.py"
+ script.write_text(
+ textwrap.dedent(
+ """
+ import sys
+
+ # Simulate git clone progress (each line > 128 chars)
+ progress_lines = [
+ "Cloning into 'repository'...\\n",
+ "remote: Enumerating objects: 1234, done.\\n",
+ "remote: Counting objects: 0% (0/1234) \\n",
+ "remote: Counting objects: 10% (123/1234) \\n",
+ "remote: Counting objects: 20% (247/1234) \\n",
+ "remote: Counting objects: 30% (370/1234) \\n",
+ "remote: Counting objects: 40% (494/1234) \\n",
+ "remote: Counting objects: 50% (617/1234) \\n",
+ "remote: Counting objects: 60% (740/1234) \\n",
+ "remote: Counting objects: 70% (864/1234) \\n",
+ "remote: Counting objects: 80% (987/1234) \\n",
+ "remote: Counting objects: 90% (1111/1234) \\n",
+ "remote: Counting objects: 100% (1234/1234), done.\\n",
+ ]
+
+ for line in progress_lines:
+ sys.stderr.write(line)
+ sys.stderr.flush()
+ """,
+ ),
+ )
+
+ # Run with progress callback
+ run(
+ [sys.executable, str(script)],
+ log_in_real_time=True,
+ callback=capture_callback,
+ )
+
+ # Check that output is fragmented
+ assert len(captured_chunks) > 0, "Should capture output"
+
+ # Join all chunks to verify content
+ full_output = "".join(captured_chunks)
+ assert "Cloning into 'repository'..." in full_output
+ assert "remote: Counting objects: 100%" in full_output
+
+ def test_real_time_streaming(self, tmp_path: pathlib.Path) -> None:
+ """Test that output is streamed in real-time, not buffered until end."""
+ captured_chunks: list[str] = []
+ capture_times: list[float] = []
+
+ def timing_callback(output: str, timestamp: datetime.datetime) -> None:
+ """Capture output and timing."""
+ captured_chunks.append(output)
+ capture_times.append(time.time())
+
+ # Create a script with delayed output
+ script = tmp_path / "streaming_test.py"
+ script.write_text(
+ textwrap.dedent(
+ """
+ import sys
+ import time
+
+ sys.stderr.write("First message\\n")
+ sys.stderr.flush()
+ time.sleep(0.1)
+
+ sys.stderr.write("Second message\\n")
+ sys.stderr.flush()
+ time.sleep(0.1)
+
+ sys.stderr.write("Third message\\n")
+ sys.stderr.flush()
+ """,
+ ),
+ )
+
+ time.time()
+ run(
+ [sys.executable, str(script)],
+ log_in_real_time=True,
+ callback=timing_callback,
+ )
+
+ # In current implementation, all stderr is read in one chunk
+ # This demonstrates the buffering issue
+ assert len(captured_chunks) >= 1, "Should get at least 1 chunk"
+
+ # Check if output was buffered (all messages in one chunk)
+ if len(captured_chunks) >= 2 and captured_chunks[-1] == "\r":
+ # Excluding the final \r
+ main_chunks = captured_chunks[:-1]
+ if len(main_chunks) == 1:
+ # All output came in one chunk - buffered, not streamed
+ assert "First message" in main_chunks[0]
+ assert "Second message" in main_chunks[0]
+ assert "Third message" in main_chunks[0]
+
+ def test_callback_with_mixed_output(self, tmp_path: pathlib.Path) -> None:
+ """Test callback with mixed stdout and stderr output."""
+ captured_chunks: list[str] = []
+
+ def capture_callback(output: str, timestamp: datetime.datetime) -> None:
+ """Capture output chunks."""
+ if output != "\r":
+ captured_chunks.append(output)
+
+ # Create a script with mixed output
+ script = tmp_path / "mixed_output_test.py"
+ script.write_text(
+ textwrap.dedent(
+ """
+ import sys
+
+ # Note: callback only captures stderr in current implementation
+ sys.stdout.write("This goes to stdout\\n")
+ sys.stdout.flush()
+
+ sys.stderr.write("This goes to stderr\\n")
+ sys.stderr.flush()
+
+ sys.stdout.write("More stdout\\n")
+ sys.stdout.flush()
+
+ sys.stderr.write("More stderr\\n")
+ sys.stderr.flush()
+ """,
+ ),
+ )
+
+ stdout_result = run(
+ [sys.executable, str(script)],
+ log_in_real_time=True,
+ callback=capture_callback,
+ )
+
+ # Verify callback only gets stderr (current behavior)
+ captured_text = "".join(captured_chunks)
+ assert "This goes to stderr" in captured_text
+ assert "More stderr" in captured_text
+ assert "This goes to stdout" not in captured_text
+
+ # Verify stdout is captured in return value
+ assert "This goes to stdout" in stdout_result
+ assert "More stdout" in stdout_result
+
+ def test_no_callback_no_streaming(self, tmp_path: pathlib.Path) -> None:
+ """Test that without callback, output is not streamed."""
+ # Create a script with output
+ script = tmp_path / "no_callback_test.py"
+ script.write_text(
+ textwrap.dedent(
+ """
+ import sys
+ sys.stderr.write("Error output\\n")
+ sys.stdout.write("Normal output\\n")
+ """,
+ ),
+ )
+
+ # Run without callback
+ output = run(
+ [sys.executable, str(script)],
+ log_in_real_time=False, # No real-time logging
+ )
+
+ # Should get stdout only
+ assert output.strip() == "Normal output"
+
+ def test_progress_callback_with_carriage_returns(
+ self,
+ tmp_path: pathlib.Path,
+ ) -> None:
+ """Test handling of carriage returns in progress output."""
+ captured_chunks: list[str] = []
+
+ def capture_callback(output: str, timestamp: datetime.datetime) -> None:
+ """Capture output chunks."""
+ captured_chunks.append(output)
+
+ # Create a script using carriage returns for progress
+ script = tmp_path / "cr_progress_test.py"
+ script.write_text(
+ textwrap.dedent(
+ """
+ import sys
+ import time
+
+ # Simulate progress with carriage returns
+ for i in range(0, 101, 20):
+ sys.stderr.write(f"\\rProgress: {i}%")
+ sys.stderr.flush()
+ time.sleep(0.01)
+
+ sys.stderr.write("\\rProgress: 100%\\n")
+ sys.stderr.flush()
+ """,
+ ),
+ )
+
+ run(
+ [sys.executable, str(script)],
+ log_in_real_time=True,
+ callback=capture_callback,
+ )
+
+ # Join chunks and check content
+ full_output = "".join(captured_chunks)
+ assert "Progress:" in full_output
+ assert "100%" in full_output
+
+ # Should have final \r
+ assert captured_chunks[-1] == "\r"
+
+
+class TestDesiredBehavior:
+ """Tests demonstrating the desired line-buffered behavior."""
+
+ def test_line_buffered_output(self, tmp_path: pathlib.Path) -> None:
+ """Test ideal behavior: output should be line-buffered, not chunk-buffered."""
+ captured_chunks: list[str] = []
+
+ def capture_callback(output: str, timestamp: datetime.datetime) -> None:
+ """Capture output chunks."""
+ if output != "\r":
+ captured_chunks.append(output)
+
+ # Create a script with git-style progress
+ script = tmp_path / "line_buffer_test.py"
+ script.write_text(
+ textwrap.dedent(
+ """
+ import sys
+
+ # Each line should come as a separate chunk
+ lines = [
+ "Starting process...\\n",
+ "Processing item 1 of 100\\r",
+ "Processing item 50 of 100\\r",
+ "Processing item 100 of 100\\r",
+ "Process complete!\\n"
+ ]
+
+ for line in lines:
+ sys.stderr.write(line)
+ sys.stderr.flush()
+ """,
+ ),
+ )
+
+ # This test will fail with current implementation
+ # but shows what we want to achieve
+ run(
+ [sys.executable, str(script)],
+ log_in_real_time=True,
+ callback=capture_callback,
+ )
+
+ # Ideally, each line should be a separate chunk
+ # Current behavior: may fragment at 128 bytes
+ # Desired behavior: each complete line is one chunk
+
+ # For now, just verify we got output
+ assert len(captured_chunks) > 0, "Should capture output"
+ full_output = "".join(captured_chunks)
+ assert "Starting process..." in full_output
+ assert "Process complete!" in full_output
+
+ def test_line_based_output_chunks(self, tmp_path: pathlib.Path) -> None:
+ """Demonstrate that output is now properly line-buffered."""
+ captured_chunks: list[str] = []
+
+ def capture_callback(output: str, timestamp: datetime.datetime) -> None:
+ """Capture output chunks."""
+ captured_chunks.append(output)
+
+ # Create a script with realistic git clone output
+ script = tmp_path / "line_based_output.py"
+ script.write_text(
+ textwrap.dedent(
+ """
+ import sys
+
+ # Simulate git clone output with multiple lines
+ output = (
+ "Cloning into '/home/user/project'...\\n"
+ "remote: Enumerating objects: 11363, done.\\n"
+ "remote: Counting objects: 0% (0/1234) \\n"
+ "remote: Counting objects: 50% (617/1234) \\n"
+ "remote: Counting objects: 100% (1234/1234), done.\\n"
+ )
+
+ sys.stderr.write(output)
+ sys.stderr.flush()
+ """,
+ ),
+ )
+
+ run(
+ [sys.executable, str(script)],
+ log_in_real_time=True,
+ callback=capture_callback,
+ )
+
+ # With line-buffered reading, we should get one chunk per line
+ non_cr_chunks = [c for c in captured_chunks if c != "\r"]
+ assert len(non_cr_chunks) == 5, (
+ f"Expected 5 chunks (one per line), got {len(non_cr_chunks)}"
+ )
+
+ # Verify each chunk is a complete line
+ for chunk in non_cr_chunks:
+ assert chunk.endswith("\n"), (
+ f"Each chunk should be a complete line: {chunk!r}"
+ )
+
+ def test_no_fragmentation_of_progress_lines(self, tmp_path: pathlib.Path) -> None:
+ """Test that progress lines should not be fragmented."""
+ captured_chunks: list[str] = []
+
+ def capture_callback(output: str, timestamp: datetime.datetime) -> None:
+ """Capture output chunks."""
+ if output != "\r":
+ captured_chunks.append(output)
+
+ # Create a script with long progress lines
+ script = tmp_path / "no_fragment_test.py"
+ script.write_text(
+ textwrap.dedent(
+ """
+ import sys
+
+ # Long progress line that would be fragmented at 128 bytes
+ long_line = (
+ "remote: Counting objects: 100% (1234567890/1234567890), "
+ "12.34 GiB | 123.45 MiB/s, done.\\n"
+ )
+ sys.stderr.write(long_line)
+ sys.stderr.flush()
+ """,
+ ),
+ )
+
+ run(
+ [sys.executable, str(script)],
+ log_in_real_time=True,
+ callback=capture_callback,
+ )
+
+ # In ideal implementation, this would be one chunk
+ # Current implementation will fragment it
+
+ # Join all chunks and verify content is preserved
+ full_output = "".join(captured_chunks)
+ assert "remote: Counting objects: 100%" in full_output
+ assert "done." in full_output
+
+
+class TestRunBehavior:
+ """Test general run() behavior."""
+
+ def test_run_with_error(self, tmp_path: pathlib.Path) -> None:
+ """Test run with non-zero exit code."""
+ script = tmp_path / "error_test.py"
+ script.write_text(
+ """
+import sys
+sys.stderr.write("Error occurred\\n")
+sys.exit(1)
+""",
+ )
+
+ with pytest.raises(exc.CommandError) as excinfo:
+ run([sys.executable, str(script)])
+
+ assert excinfo.value.returncode == 1
+ assert "Error occurred" in str(excinfo.value.output)
+
+ def test_run_check_returncode_false(self, tmp_path: pathlib.Path) -> None:
+ """Test run with check_returncode=False doesn't raise on error."""
+ script = tmp_path / "error_test.py"
+ script.write_text(
+ """
+import sys
+sys.stderr.write("Error output\\n")
+sys.exit(1)
+""",
+ )
+
+ # Should not raise
+ output = run(
+ [sys.executable, str(script)],
+ check_returncode=False,
+ )
+
+ # With error, output comes from stderr
+ assert "Error output" in output
+
+
+class TestFlushingBehavior:
+ """Test that output is properly flushed and streamed."""
+
+ def test_verify_flushing_with_timing(self, tmp_path: pathlib.Path) -> None:
+ """Verify that output is flushed immediately, not buffered."""
+ captured_data: list[tuple[str, float]] = []
+
+ def timing_callback(output: str, timestamp: datetime.datetime) -> None:
+ """Capture output with precise timing."""
+ captured_data.append((output, time.time()))
+
+ # Create a script that outputs with delays
+ script = tmp_path / "flush_timing_test.py"
+ script.write_text(
+ textwrap.dedent(
+ """
+ import sys
+ import time
+
+ start_time = time.time()
+
+ # Output with explicit timing
+ sys.stderr.write("Start\\n")
+ sys.stderr.flush()
+ time.sleep(0.1)
+
+ sys.stderr.write("Middle\\n")
+ sys.stderr.flush()
+ time.sleep(0.1)
+
+ sys.stderr.write("End\\n")
+ sys.stderr.flush()
+ """,
+ ),
+ )
+
+ run(
+ [sys.executable, str(script)],
+ log_in_real_time=True,
+ callback=timing_callback,
+ )
+
+ # Verify we got output
+ assert len(captured_data) > 0
+
+ # Check if timing indicates buffering
+ # If all output comes at once, times will be very close
+ if len(captured_data) >= 2:
+ non_cr_data = [(o, t) for o, t in captured_data if o != "\r"]
+ if len(non_cr_data) >= 2:
+ time_diff = non_cr_data[-1][1] - non_cr_data[0][1]
+ # If properly streamed, should have ~0.2s difference
+ # If buffered, all comes at once (< 0.05s)
+ # Current implementation may buffer
+ assert time_diff >= 0 # Just verify we got timing data
+
+ def test_unbuffered_subprocess_output(self, tmp_path: pathlib.Path) -> None:
+ """Test that subprocess with unbuffered output works correctly."""
+ captured_chunks: list[str] = []
+
+ def capture_callback(output: str, timestamp: datetime.datetime) -> None:
+ """Capture output chunks."""
+ captured_chunks.append(output)
+
+ # Create an unbuffered Python script
+ script = tmp_path / "unbuffered_test.py"
+ script.write_text(
+ textwrap.dedent(
+ """
+ import sys
+ import time
+
+ # Force unbuffered output
+ sys.stderr = sys.stderr.detach()
+ sys.stderr = io.TextIOWrapper(sys.stderr, write_through=True)
+
+ for i in range(3):
+ sys.stderr.write(f"Line {i}\\n")
+ sys.stderr.flush()
+ time.sleep(0.05)
+ """,
+ ),
+ )
+
+ # Add io import to script
+ script.write_text(
+ textwrap.dedent(
+ """
+ import sys
+ import time
+ import io
+
+ # Force unbuffered output
+ sys.stderr = sys.stderr.detach()
+ sys.stderr = io.TextIOWrapper(sys.stderr, write_through=True)
+
+ for i in range(3):
+ sys.stderr.write(f"Line {i}\\n")
+ sys.stderr.flush()
+ time.sleep(0.05)
+ """,
+ ),
+ )
+
+ run(
+ [sys.executable, "-u", str(script)], # -u for unbuffered
+ log_in_real_time=True,
+ callback=capture_callback,
+ )
+
+ # Verify output was captured
+ full_output = "".join(c for c in captured_chunks if c != "\r")
+ assert "Line 0" in full_output
+ assert "Line 1" in full_output
+ assert "Line 2" in full_output
+
+ def test_carriage_return_overwrites(self, tmp_path: pathlib.Path) -> None:
+ """Test that carriage returns properly overwrite previous output."""
+ captured_chunks: list[str] = []
+
+ def capture_callback(output: str, timestamp: datetime.datetime) -> None:
+ """Capture output chunks."""
+ captured_chunks.append(output)
+
+ # Script that uses \r to overwrite
+ script = tmp_path / "overwrite_test.py"
+ script.write_text(
+ textwrap.dedent(
+ """
+ import sys
+ import time
+
+ # Progress that overwrites itself
+ sys.stderr.write("Progress: 0%\\r")
+ sys.stderr.flush()
+ time.sleep(0.01)
+
+ sys.stderr.write("Progress: 50%\\r")
+ sys.stderr.flush()
+ time.sleep(0.01)
+
+ sys.stderr.write("Progress: 100%\\n")
+ sys.stderr.flush()
+ """,
+ ),
+ )
+
+ run(
+ [sys.executable, str(script)],
+ log_in_real_time=True,
+ callback=capture_callback,
+ )
+
+ # Join all output
+ full_output = "".join(captured_chunks)
+
+ # Should contain progress updates
+ assert "Progress:" in full_output
+ assert "100%" in full_output
+
+ # Verify \r was passed through
+ assert "\r" in full_output or captured_chunks[-1] == "\r"
+
+ def test_ansi_escape_sequences(self, tmp_path: pathlib.Path) -> None:
+ """Test that ANSI escape sequences are preserved."""
+ captured_chunks: list[str] = []
+
+ def capture_callback(output: str, timestamp: datetime.datetime) -> None:
+ """Capture output chunks."""
+ captured_chunks.append(output)
+
+ # Script with ANSI escape codes
+ script = tmp_path / "ansi_test.py"
+ script.write_text(
+ textwrap.dedent(
+ """
+ import sys
+
+ # ANSI escape sequences for colors and cursor control
+ sys.stderr.write("\\033[32mGreen text\\033[0m\\n") # Green color
+ sys.stderr.write("\\033[1mBold text\\033[0m\\n") # Bold
+ sys.stderr.write("\\033[2KClearing line\\r") # Clear line
+ sys.stderr.write("Final output\\n")
+ sys.stderr.flush()
+ """,
+ ),
+ )
+
+ run(
+ [sys.executable, str(script)],
+ log_in_real_time=True,
+ callback=capture_callback,
+ )
+
+ # Join output
+ full_output = "".join(c for c in captured_chunks if c != "\r")
+
+ # Verify ANSI codes are preserved
+ assert "\033[32m" in full_output or "\\033[32m" in full_output # Green
+ assert "\033[1m" in full_output or "\\033[1m" in full_output # Bold
+ assert "\033[2K" in full_output or "\\033[2K" in full_output # Clear line
+ assert "Final output" in full_output
+
+ def test_git_style_progress_simulation(self, tmp_path: pathlib.Path) -> None:
+ """Test git-style progress output with percentages and carriage returns."""
+ captured_chunks: list[str] = []
+
+ def capture_callback(output: str, timestamp: datetime.datetime) -> None:
+ """Capture output chunks."""
+ captured_chunks.append(output)
+
+ # Simulate git clone-style progress
+ script = tmp_path / "git_style_progress.py"
+ script.write_text(
+ textwrap.dedent(
+ """
+ import sys
+ import time
+
+ # Simulate git clone output
+ sys.stderr.write("Cloning into 'repo'...\\n")
+ sys.stderr.flush()
+
+ # Progress with carriage returns
+ for i in range(0, 101, 10):
+ sys.stderr.write(f"\\rReceiving objects: {i:3d}% ({i}/100)")
+ sys.stderr.flush()
+ time.sleep(0.01)
+
+ sys.stderr.write("\\rReceiving objects: 100% (100/100), done.\\n")
+ sys.stderr.flush()
+
+ # Resolving deltas
+ for i in range(0, 101, 25):
+ sys.stderr.write(f"\\rResolving deltas: {i:3d}% ({i}/100)")
+ sys.stderr.flush()
+ time.sleep(0.01)
+
+ sys.stderr.write("\\rResolving deltas: 100% (100/100), done.\\n")
+ sys.stderr.flush()
+ """,
+ ),
+ )
+
+ run(
+ [sys.executable, str(script)],
+ log_in_real_time=True,
+ callback=capture_callback,
+ )
+
+ # Verify output
+ full_output = "".join(captured_chunks)
+ assert "Cloning into 'repo'..." in full_output
+ assert "Receiving objects:" in full_output
+ assert "Resolving deltas:" in full_output
+ assert "done." in full_output
+
+ def test_mixed_line_endings(self, tmp_path: pathlib.Path) -> None:
+ r"""Test handling of mixed line endings (\n, \r, \r\n)."""
+ captured_chunks: list[str] = []
+
+ def capture_callback(output: str, timestamp: datetime.datetime) -> None:
+ """Capture output chunks."""
+ captured_chunks.append(output)
+
+ script = tmp_path / "mixed_endings_test.py"
+ script.write_text(
+ textwrap.dedent(
+ """
+ import sys
+
+ # Different line endings
+ sys.stderr.write("Unix line\\n")
+ sys.stderr.write("Mac line\\r")
+ sys.stderr.write("Windows line\\r\\n")
+ sys.stderr.write("Progress 50%\\r")
+ sys.stderr.write("Progress 100%\\n")
+ sys.stderr.flush()
+ """,
+ ),
+ )
+
+ run(
+ [sys.executable, str(script)],
+ log_in_real_time=True,
+ callback=capture_callback,
+ )
+
+ # Verify all line types are captured
+ full_output = "".join(captured_chunks)
+ assert "Unix line" in full_output
+ assert "Mac line" in full_output
+ assert "Windows line" in full_output
+ assert "Progress 100%" in full_output
+
+ def test_large_output_streaming(self, tmp_path: pathlib.Path) -> None:
+ """Test streaming of large output without blocking."""
+ captured_size = 0
+
+ def size_callback(output: str, timestamp: datetime.datetime) -> None:
+ """Track output size."""
+ nonlocal captured_size
+ captured_size += len(output)
+
+ # Generate large output
+ script = tmp_path / "large_output_test.py"
+ script.write_text(
+ textwrap.dedent(
+ """
+ import sys
+
+ # Generate 10KB of output
+ for i in range(100):
+ line = f"Line {i:04d}: " + "X" * 90 + "\\n"
+ sys.stderr.write(line)
+ if i % 10 == 0:
+ sys.stderr.flush()
+
+ sys.stderr.flush()
+ """,
+ ),
+ )
+
+ run(
+ [sys.executable, str(script)],
+ log_in_real_time=True,
+ callback=size_callback,
+ )
+
+ # Verify we captured significant output
+ assert captured_size > 9000 # Should be ~10KB
+
+ def test_immediate_flush_verification(self, tmp_path: pathlib.Path) -> None:
+ """Verify that explicit flushes in subprocess are honored."""
+ capture_events: list[tuple[str, float]] = []
+
+ def event_callback(output: str, timestamp: datetime.datetime) -> None:
+ """Capture output events with timing."""
+ capture_events.append((output, time.time()))
+
+ script = tmp_path / "explicit_flush_test.py"
+ script.write_text(
+ textwrap.dedent(
+ """
+ import sys
+ import time
+
+ # Test explicit flush behavior
+ sys.stderr.write("Before flush")
+ # No flush - may be buffered
+ time.sleep(0.05)
+
+ sys.stderr.write(" - After delay\\n")
+ sys.stderr.flush() # Explicit flush
+
+ time.sleep(0.05)
+
+ sys.stderr.write("Second line\\n")
+ sys.stderr.flush()
+ """,
+ ),
+ )
+
+ run(
+ [sys.executable, str(script)],
+ log_in_real_time=True,
+ callback=event_callback,
+ )
+
+ # Verify output was captured
+ full_output = "".join(e[0] for e in capture_events if e[0] != "\r")
+ assert "Before flush" in full_output
+ assert "After delay" in full_output
+ assert "Second line" in full_output
+
+ def test_stderr_vs_stdout_separation(self, tmp_path: pathlib.Path) -> None:
+ """Test that stderr is captured by callback while stdout is returned."""
+ stderr_chunks: list[str] = []
+
+ def stderr_callback(output: str, timestamp: datetime.datetime) -> None:
+ """Capture stderr output."""
+ if output != "\r":
+ stderr_chunks.append(output)
+
+ script = tmp_path / "stream_separation_test.py"
+ script.write_text(
+ textwrap.dedent(
+ """
+ import sys
+
+ # Write to both streams
+ sys.stdout.write("STDOUT: Line 1\\n")
+ sys.stdout.write("STDOUT: Line 2\\n")
+ sys.stdout.flush()
+
+ sys.stderr.write("STDERR: Progress 1\\n")
+ sys.stderr.write("STDERR: Progress 2\\n")
+ sys.stderr.flush()
+ """,
+ ),
+ )
+
+ stdout_result = run(
+ [sys.executable, str(script)],
+ log_in_real_time=True,
+ callback=stderr_callback,
+ )
+
+ # Verify separation
+ stderr_output = "".join(stderr_chunks)
+ assert "STDERR: Progress 1" in stderr_output
+ assert "STDERR: Progress 2" in stderr_output
+ assert "STDOUT:" not in stderr_output
+
+ assert "STDOUT: Line 1" in stdout_result
+ assert "STDOUT: Line 2" in stdout_result
+ assert "STDERR:" not in stdout_result
+
+
+class TestStreamingFix:
+ """Test the fix for proper line-buffered streaming."""
+
+ def test_line_by_line_streaming(self, tmp_path: pathlib.Path) -> None:
+ """Test that output is streamed line by line, not in fixed chunks."""
+ captured_chunks: list[str] = []
+
+ def capture_callback(output: str, timestamp: datetime.datetime) -> None:
+ """Capture output chunks."""
+ captured_chunks.append(output)
+
+ # Create a script that outputs lines of varying lengths
+ script = tmp_path / "line_streaming_test.py"
+ script.write_text(
+ textwrap.dedent(
+ """
+ import sys
+ import time
+
+ # Output lines of different lengths to verify line-buffered behavior
+ lines = [
+ "Short line\\n",
+ "This is a medium length line that is longer than the short one\\n",
+ "This is a very long line that should definitely be longer than "
+ "128 characters to ensure we're not just getting lucky with the "
+ "buffering behavior\\n",
+ "Another short\\n",
+ "Progress: 50% [=================> ]\\r",
+ "Progress: 100% [======================================]\\n",
+ ]
+
+ for line in lines:
+ sys.stderr.write(line)
+ sys.stderr.flush()
+ time.sleep(0.01) # Small delay to ensure separate chunks
+ """,
+ ),
+ )
+
+ # Run with the fixed implementation
+ run(
+ [sys.executable, str(script)],
+ log_in_real_time=True,
+ callback=capture_callback,
+ )
+
+ # Remove the final \r if present
+ non_cr_chunks = [c for c in captured_chunks if c != "\r"]
+
+ # With proper line buffering, each line should be its own chunk
+ # This test will fail with current implementation but pass with the fix
+ assert len(non_cr_chunks) >= 6, (
+ f"Expected at least 6 chunks (one per line), got {len(non_cr_chunks)}"
+ )
+
+ # Verify that complete lines are preserved
+ full_output = "".join(non_cr_chunks)
+ assert "Short line\n" in full_output
+ assert "Progress: 100%" in full_output
+
+ # Check that lines weren't fragmented
+ # With line buffering, no line should be split across chunks
+ for chunk in non_cr_chunks:
+ # Each chunk should either end with \n or \r (except possibly the last)
+ if chunk and chunk != non_cr_chunks[-1]:
+ assert chunk.endswith(("\n", "\r")), (
+ f"Chunk should end with newline or carriage return: {chunk!r}"
+ )
+
+ def test_real_time_line_streaming_with_timing(self, tmp_path: pathlib.Path) -> None:
+ """Test that lines are streamed immediately when flushed, not buffered."""
+ capture_events: list[tuple[str, float]] = []
+ start_time = time.time()
+
+ def timing_callback(output: str, timestamp: datetime.datetime) -> None:
+ """Capture output with relative timing."""
+ capture_events.append((output, time.time() - start_time))
+
+ script = tmp_path / "realtime_line_test.py"
+ script.write_text(
+ textwrap.dedent(
+ """
+ import sys
+ import time
+
+ # Output lines with delays to verify real-time streaming
+ sys.stderr.write("Line 1: Starting\\n")
+ sys.stderr.flush()
+ time.sleep(0.1)
+
+ sys.stderr.write("Line 2: After 100ms\\n")
+ sys.stderr.flush()
+ time.sleep(0.1)
+
+ sys.stderr.write("Line 3: After 200ms\\n")
+ sys.stderr.flush()
+ """,
+ ),
+ )
+
+ run(
+ [sys.executable, str(script)],
+ log_in_real_time=True,
+ callback=timing_callback,
+ )
+
+ # Remove final \r
+ events = [(o, t) for o, t in capture_events if o != "\r"]
+
+ # With proper streaming, we should get 3 separate events
+ assert len(events) >= 3, f"Expected at least 3 events, got {len(events)}"
+
+ # Check timing - with line buffering, lines should come separately
+ if len(events) >= 3:
+ # Verify we got separate events for each line
+ assert events[0][0] == "Line 1: Starting\n"
+ assert events[1][0] == "Line 2: After 100ms\n"
+ assert events[2][0] == "Line 3: After 200ms\n"
+
+ # The timing might vary due to process startup and buffering,
+ # but we should see that lines come as separate events
+ # rather than all at once
+
+ def test_no_fragmentation_with_line_buffering(self, tmp_path: pathlib.Path) -> None:
+ """Test that line buffering prevents fragmentation of lines."""
+ captured_chunks: list[str] = []
+
+ def capture_callback(output: str, timestamp: datetime.datetime) -> None:
+ """Capture output chunks."""
+ if output != "\r":
+ captured_chunks.append(output)
+
+ # Create a script with lines longer than 128 bytes
+ script = tmp_path / "no_fragment_line_test.py"
+ long_line = "A" * 200 # 200 chars, definitely > 128 bytes
+ script.write_text(
+ f"""
+import sys
+
+# Output a long line that would be fragmented with fixed-size buffering
+sys.stderr.write("Start: {long_line} :End\\n")
+sys.stderr.flush()
+
+# Output another long line
+sys.stderr.write("Line2: {"B" * 150} :Done\\n")
+sys.stderr.flush()
+""",
+ )
+
+ run(
+ [sys.executable, str(script)],
+ log_in_real_time=True,
+ callback=capture_callback,
+ )
+
+ # With line buffering, long lines should not be fragmented
+ # Each line should be in a single chunk
+ assert len(captured_chunks) == 2, (
+ f"Expected 2 chunks (one per line), got {len(captured_chunks)}"
+ )
+
+ # Verify complete lines
+ assert captured_chunks[0].startswith("Start: ")
+ assert captured_chunks[0].endswith(" :End\n")
+ assert "A" * 200 in captured_chunks[0]
+
+ assert captured_chunks[1].startswith("Line2: ")
+ assert captured_chunks[1].endswith(" :Done\n")
+ assert "B" * 150 in captured_chunks[1]
+
+
+class TestRealWorldScenarios:
+ """Test real-world subprocess output scenarios."""
+
+ def test_npm_style_progress(self, tmp_path: pathlib.Path) -> None:
+ """Test npm-style progress with spinner and percentages."""
+ captured_chunks: list[str] = []
+
+ def capture_callback(output: str, timestamp: datetime.datetime) -> None:
+ """Capture output chunks."""
+ captured_chunks.append(output)
+
+ # Simulate npm install progress
+ script = tmp_path / "npm_progress.py"
+ script.write_text(
+ textwrap.dedent(
+ """
+ import sys
+ import time
+
+ # npm-style progress
+ spinners = ['⠋', '⠙', '⠹', '⠸', '⠼', '⠴', '⠦', '⠧', '⠇', '⠏']
+
+ sys.stderr.write("Installing dependencies...\\n")
+ sys.stderr.flush()
+
+ for i in range(20):
+ spinner = spinners[i % len(spinners)]
+ percent = (i + 1) * 5
+ sys.stderr.write(f"\\r{spinner} Progress: {percent}%")
+ sys.stderr.flush()
+ time.sleep(0.01)
+
+ sys.stderr.write("\\r✓ Dependencies installed\\n")
+ sys.stderr.flush()
+ """,
+ ),
+ )
+
+ run(
+ [sys.executable, str(script)],
+ log_in_real_time=True,
+ callback=capture_callback,
+ )
+
+ full_output = "".join(captured_chunks)
+ assert "Installing dependencies" in full_output
+ assert "Progress:" in full_output
+ assert "✓ Dependencies installed" in full_output
+
+ def test_long_running_with_periodic_updates(self, tmp_path: pathlib.Path) -> None:
+ """Test long-running process with periodic status updates."""
+ captured_chunks: list[str] = []
+ timestamps: list[float] = []
+
+ def timing_callback(output: str, timestamp: datetime.datetime) -> None:
+ """Capture output with timing."""
+ captured_chunks.append(output)
+ timestamps.append(time.time())
+
+ script = tmp_path / "long_running.py"
+ script.write_text(
+ textwrap.dedent(
+ """
+ import sys
+ import time
+
+ # Simulate a long-running process
+ stages = [
+ "Initializing...",
+ "Loading configuration...",
+ "Connecting to database...",
+ "Processing data...",
+ "Finalizing..."
+ ]
+
+ for i, stage in enumerate(stages):
+ sys.stderr.write(f"\\r[{i+1}/{len(stages)}] {stage}")
+ sys.stderr.flush()
+ time.sleep(0.02) # Simulate work
+
+ sys.stderr.write("\\r[Done] Process completed successfully\\n")
+ sys.stderr.flush()
+ """,
+ ),
+ )
+
+ run(
+ [sys.executable, str(script)],
+ log_in_real_time=True,
+ callback=timing_callback,
+ )
+
+ full_output = "".join(captured_chunks)
+ assert "Initializing" in full_output
+ assert "Process completed successfully" in full_output
+
+ def test_multiline_progress_bars(self, tmp_path: pathlib.Path) -> None:
+ """Test handling of multi-line progress displays."""
+ captured_chunks: list[str] = []
+
+ def capture_callback(output: str, timestamp: datetime.datetime) -> None:
+ """Capture output chunks."""
+ captured_chunks.append(output)
+
+ script = tmp_path / "multiline_progress.py"
+ script.write_text(
+ textwrap.dedent(
+ """
+ import sys
+
+ # Simulate multi-line progress (like docker pull)
+ sys.stderr.write("Pulling image layers:\\n")
+ sys.stderr.write("layer1: Downloading [==> ] 20%\\r")
+ sys.stderr.flush()
+ sys.stderr.write("layer1: Downloading [====> ] 40%\\r")
+ sys.stderr.flush()
+ sys.stderr.write("layer1: Downloading [======> ] 60%\\r")
+ sys.stderr.flush()
+ sys.stderr.write("layer1: Downloading [========>] 80%\\r")
+ sys.stderr.flush()
+ sys.stderr.write("layer1: Download complete\\n")
+ sys.stderr.flush()
+ """,
+ ),
+ )
+
+ run(
+ [sys.executable, str(script)],
+ log_in_real_time=True,
+ callback=capture_callback,
+ )
+
+ full_output = "".join(captured_chunks)
+ assert "Pulling image layers" in full_output
+ assert "Downloading" in full_output
+ assert "Download complete" in full_output
+
+ def test_binary_output_handling(self, tmp_path: pathlib.Path) -> None:
+ """Test handling of binary/non-UTF8 output."""
+ captured_chunks: list[str] = []
+ had_errors = False
+
+ def capture_callback(output: str, timestamp: datetime.datetime) -> None:
+ """Capture output chunks."""
+ nonlocal had_errors
+ try:
+ captured_chunks.append(output)
+ except Exception:
+ had_errors = True
+
+ script = tmp_path / "binary_output.py"
+ script.write_text(
+ textwrap.dedent(
+ """
+ import sys
+
+ # Mix text and special characters
+ sys.stderr.write("Normal text\\n")
+ sys.stderr.write("Special chars: © ® ™\\n")
+ sys.stderr.write("Emoji: 🚀 ✨ 🎉\\n")
+ sys.stderr.flush()
+ """,
+ ),
+ )
+
+ run(
+ [sys.executable, str(script)],
+ log_in_real_time=True,
+ callback=capture_callback,
+ )
+
+ assert not had_errors
+ full_output = "".join(captured_chunks)
+ assert "Normal text" in full_output
+ assert "Special chars" in full_output
+ assert "Emoji" in full_output
+
+ def test_interleaved_output_timing(self, tmp_path: pathlib.Path) -> None:
+ """Test timing of interleaved stdout/stderr output."""
+ script = tmp_path / "interleaved.py"
+ script.write_text(
+ textwrap.dedent(
+ """
+ import sys
+ import time
+
+ # Interleave stdout and stderr
+ for i in range(3):
+ sys.stdout.write(f"stdout {i}\\n")
+ sys.stdout.flush()
+ sys.stderr.write(f"stderr {i}\\n")
+ sys.stderr.flush()
+ time.sleep(0.01)
+ """,
+ ),
+ )
+
+ stderr_chunks: list[str] = []
+
+ def capture_stderr(output: str, timestamp: datetime.datetime) -> None:
+ """Capture only stderr."""
+ if output != "\r":
+ stderr_chunks.append(output)
+
+ stdout_result = run(
+ [sys.executable, str(script)],
+ log_in_real_time=True,
+ callback=capture_stderr,
+ )
+
+ # Verify correct stream separation
+ stderr_text = "".join(stderr_chunks)
+ assert "stderr 0" in stderr_text
+ assert "stderr 1" in stderr_text
+ assert "stderr 2" in stderr_text
+ assert "stdout" not in stderr_text
+
+ assert "stdout 0" in stdout_result
+ assert "stdout 1" in stdout_result
+ assert "stdout 2" in stdout_result
+ assert "stderr" not in stdout_result
+
+
+class TestEdgeCases:
+ """Test edge cases and error conditions."""
+
+ def test_empty_output(self, tmp_path: pathlib.Path) -> None:
+ """Test handling of subprocess with no output."""
+ captured_chunks: list[str] = []
+
+ def capture_callback(output: str, timestamp: datetime.datetime) -> None:
+ """Capture output chunks."""
+ captured_chunks.append(output)
+
+ script = tmp_path / "empty_output.py"
+ script.write_text("# No output")
+
+ run(
+ [sys.executable, str(script)],
+ log_in_real_time=True,
+ callback=capture_callback,
+ )
+
+ # Should only get the final \r
+ assert len(captured_chunks) == 1
+ assert captured_chunks[0] == "\r"
+
+ def test_only_stdout_no_stderr(self, tmp_path: pathlib.Path) -> None:
+ """Test subprocess that only writes to stdout."""
+ captured_chunks: list[str] = []
+
+ def capture_callback(output: str, timestamp: datetime.datetime) -> None:
+ """Capture output chunks."""
+ captured_chunks.append(output)
+
+ script = tmp_path / "stdout_only.py"
+ script.write_text(
+ """
+import sys
+sys.stdout.write("Only stdout output\\n")
+sys.stdout.flush()
+""",
+ )
+
+ stdout_result = run(
+ [sys.executable, str(script)],
+ log_in_real_time=True,
+ callback=capture_callback,
+ )
+
+ # Callback only captures stderr, so should be empty except for \r
+ assert len(captured_chunks) == 1
+ assert captured_chunks[0] == "\r"
+
+ # stdout should be captured in return value
+ assert "Only stdout output" in stdout_result
+
+ def test_very_long_lines(self, tmp_path: pathlib.Path) -> None:
+ """Test handling of very long lines that exceed buffer size."""
+ captured_chunks: list[str] = []
+
+ def capture_callback(output: str, timestamp: datetime.datetime) -> None:
+ """Capture output chunks."""
+ if output != "\r":
+ captured_chunks.append(output)
+
+ script = tmp_path / "long_lines.py"
+ script.write_text(
+ """
+import sys
+
+# Generate a very long line (> 128 bytes)
+long_line = "A" * 500 + "\\n"
+sys.stderr.write(long_line)
+sys.stderr.flush()
+
+# Another long line with progress
+progress_line = "Progress: " + "=" * 200 + "> 100%\\n"
+sys.stderr.write(progress_line)
+sys.stderr.flush()
+""",
+ )
+
+ run(
+ [sys.executable, str(script)],
+ log_in_real_time=True,
+ callback=capture_callback,
+ )
+
+ # Verify long lines are captured (may be fragmented)
+ full_output = "".join(captured_chunks)
+ assert "A" * 500 in full_output
+ assert "=" * 200 in full_output
+
+ def test_rapid_output_bursts(self, tmp_path: pathlib.Path) -> None:
+ """Test handling of rapid output bursts."""
+ captured_count = 0
+
+ def count_callback(output: str, timestamp: datetime.datetime) -> None:
+ """Count callback invocations."""
+ nonlocal captured_count
+ captured_count += 1
+
+ script = tmp_path / "rapid_output.py"
+ script.write_text(
+ """
+import sys
+
+# Rapid burst of output
+for i in range(100):
+ sys.stderr.write(f"Line {i}\\n")
+
+sys.stderr.flush()
+""",
+ )
+
+ run(
+ [sys.executable, str(script)],
+ log_in_real_time=True,
+ callback=count_callback,
+ )
+
+ # Should have received multiple callbacks
+ assert captured_count >= 2 # At least initial output + final \r
+
+ def test_callback_exception_handling(self, tmp_path: pathlib.Path) -> None:
+ """Test that exceptions in callback don't break execution."""
+ call_count = 0
+
+ def failing_callback(output: str, timestamp: datetime.datetime) -> None:
+ """Raise exception on second call."""
+ nonlocal call_count
+ call_count += 1
+ if call_count == 2:
+ msg = "Test exception"
+ raise ValueError(msg)
+
+ script = tmp_path / "normal_output.py"
+ script.write_text(
+ """
+import sys
+sys.stderr.write("Line 1\\n")
+sys.stderr.write("Line 2\\n")
+sys.stderr.flush()
+""",
+ )
+
+ # Should not raise even if callback fails
+ with contextlib.suppress(ValueError):
+ run(
+ [sys.executable, str(script)],
+ log_in_real_time=True,
+ callback=failing_callback,
+ )
+
+ # Verify callback was called multiple times
+ assert call_count >= 1
pFad - Phonifier reborn
Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.
Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.
Alternative Proxies:
Alternative Proxy
pFad Proxy
pFad v3 Proxy
pFad v4 Proxy