sketch: fixed prototype generation when included libraries contains very long lines (#2935)

facchinm · cmaglie · web-flow · commit 4891dedb4934 · 2025-06-26T17:50:21.000+02:00
* ctags: filterSketchSource: enlarge buffer size to handle long lines Fixes arduino/ArduinoCore-zephyr#140 * Added unti tests * Fixed linter warning --------- Co-authored-by: Cristian Maglie <c.maglie@arduino.cc>
diff --git a/internal/arduino/builder/internal/preprocessor/ctags.go b/internal/arduino/builder/internal/preprocessor/ctags.go
@@ -74,7 +74,7 @@ func PreprocessSketchWithCtags(
 	}
 
 	if src, err := ctagsTarget.ReadFile(); err == nil {
-		filteredSource := filterSketchSource(sketch, bytes.NewReader(src), false)
+		filteredSource := filterSketchSource(sketch, bytes.NewReader(src), false, stderr)
 		if err := ctagsTarget.WriteFile([]byte(filteredSource)); err != nil {
 			return &Result{args: result.Args(), stdout: stdout.Bytes(), stderr: stderr.Bytes()}, err
 		}
@@ -208,7 +208,7 @@ func RunCTags(ctx context.Context, sourceFile *paths.Path, buildProperties *prop
 	return stdout, stderr, args, err
 }
 
-func filterSketchSource(sketch *sketch.Sketch, source io.Reader, removeLineMarkers bool) string {
+func filterSketchSource(sketch *sketch.Sketch, source io.Reader, removeLineMarkers bool, stderr *bytes.Buffer) string {
 	fileNames := paths.NewPathList()
 	fileNames.Add(sketch.MainFile)
 	fileNames.AddAll(sketch.OtherSketchFiles)
@@ -217,6 +217,11 @@ func filterSketchSource(sketch *sketch.Sketch, source io.Reader, removeLineMarke
 	filtered := ""
 
 	scanner := bufio.NewScanner(source)
+
+	var buf []byte
+	const maxTokenSize = 1024 * 1024 // 1 MB
+	scanner.Buffer(buf, maxTokenSize)
+
 	for scanner.Scan() {
 		line := scanner.Text()
 		if filename := cpp.ParseLineMarker(line); filename != nil {
@@ -230,6 +235,11 @@ func filterSketchSource(sketch *sketch.Sketch, source io.Reader, removeLineMarke
 			filtered += line + "\n"
 		}
 	}
+	if errors.Is(scanner.Err(), bufio.ErrTooLong) {
+		fmt.Fprintf(stderr, "%s: %s",
+			i18n.Tr("An error occurred adding prototypes"),
+			i18n.Tr("line too long\n"))
+	}
 
 	return filtered
 }
diff --git a/internal/arduino/builder/internal/preprocessor/ctags_test.go b/internal/arduino/builder/internal/preprocessor/ctags_test.go
@@ -0,0 +1,42 @@
+// This file is part of arduino-cli.
+//
+// Copyright 2025 ARDUINO SA (http://www.arduino.cc/)
+//
+// This software is released under the GNU General Public License version 3,
+// which covers the main part of arduino-cli.
+// The terms of this license can be found at:
+// https://www.gnu.org/licenses/gpl-3.0.en.html
+//
+// You can be released from the requirements of the above licenses by purchasing
+// a commercial license. Buying such a license is mandatory if you want to
+// modify or otherwise use the software for commercial activities involving the
+// Arduino software without disclosing the source code of your own applications.
+// To purchase a commercial license, send an email to license@arduino.cc.
+
+package preprocessor
+
+import (
+	"bytes"
+	"fmt"
+	"testing"
+
+	"github.com/arduino/arduino-cli/internal/arduino/sketch"
+	"github.com/arduino/go-paths-helper"
+	"github.com/stretchr/testify/require"
+)
+
+func TestCtagsSketchFilter(t *testing.T) {
+	sourcePath := paths.New("testdata", "sketch_merged.cpp")
+	f, err := sourcePath.Open()
+	require.NoError(t, err)
+	t.Cleanup(func() { f.Close() })
+
+	sketch := &sketch.Sketch{
+		MainFile: paths.New("/home/megabug/Arduino/Test/Test.ino"),
+	}
+	stderr := bytes.NewBuffer(nil)
+	res := filterSketchSource(sketch, f, false, stderr)
+	require.Empty(t, stderr)
+	require.NotEmpty(t, res)
+	fmt.Println(res)
+}
diff --git a/internal/arduino/builder/internal/preprocessor/testdata/sketch_merged.cpp b/internal/arduino/builder/internal/preprocessor/testdata/sketch_merged.cpp
