Skip to content

Commit 891e730

Browse files
cmagliefacchinm
authored and
umbynos
committed
Fixed sketchbook+bootloader hex merger when bootloader is in bin format (arduino#744)
* Fixed skethbook+bootloader hex merger. Previous merger doesn't take into account .bin bootloaders and it may have problems if .hex data is not sorted by address. * output a merged sketch+bootloader bin file too * Replace logger with utils.LogIfVerbose Co-authored-by: Martino Facchin <m.facchin@arduino.cc>
1 parent ab55cb5 commit 891e730

File tree

4 files changed

+174
-63
lines changed

4 files changed

+174
-63
lines changed

‎go.mod

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ require (
66
bou.ke/monkey v1.0.1
77
github.com/GeertJohan/go.rice v1.0.0
88
github.com/arduino/board-discovery v0.0.0-20180823133458-1ba29327fb0c
9-
github.com/arduino/go-paths-helper v1.0.1
9+
github.com/arduino/go-paths-helper v1.2.0
1010
github.com/arduino/go-properties-orderedmap v1.0.0
1111
github.com/arduino/go-timeutils v0.0.0-20171220113728-d1dd9e313b1b
1212
github.com/arduino/go-win32-utils v0.0.0-20180330194947-ed041402e83b
@@ -25,6 +25,7 @@ require (
2525
github.com/juju/loggo v0.0.0-20190526231331-6e530bcce5d8 // indirect
2626
github.com/juju/testing v0.0.0-20190429233213-dfc56b8c09fc // indirect
2727
github.com/leonelquinteros/gotext v1.4.0
28+
github.com/marcinbor85/gohex v0.0.0-20200531163658-baab2527a9a2
2829
github.com/mattn/go-colorable v0.1.2
2930
github.com/mattn/go-runewidth v0.0.2 // indirect
3031
github.com/miekg/dns v1.0.5 // indirect

‎go.sum

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,8 @@ github.com/arduino/board-discovery v0.0.0-20180823133458-1ba29327fb0c h1:agh2JT9
1616
github.com/arduino/board-discovery v0.0.0-20180823133458-1ba29327fb0c/go.mod h1:HK7SpkEax/3P+0w78iRQx1sz1vCDYYw9RXwHjQTB5i8=
1717
github.com/arduino/go-paths-helper v1.0.1 h1:utYXLM2RfFlc9qp/MJTIYp3t6ux/xM6mWjeEb/WLK4Q=
1818
github.com/arduino/go-paths-helper v1.0.1/go.mod h1:HpxtKph+g238EJHq4geEPv9p+gl3v5YYu35Yb+w31Ck=
19+
github.com/arduino/go-paths-helper v1.2.0 h1:qDW93PR5IZUN/jzO4rCtexiwF8P4OIcOmcSgAYLZfY4=
20+
github.com/arduino/go-paths-helper v1.2.0/go.mod h1:HpxtKph+g238EJHq4geEPv9p+gl3v5YYu35Yb+w31Ck=
1921
github.com/arduino/go-properties-orderedmap v1.0.0 h1:caaM25TQZKkytoKQUsgqtOVbrM5i8Gb427JmW0KL05s=
2022
github.com/arduino/go-properties-orderedmap v1.0.0/go.mod h1:DKjD2VXY/NZmlingh4lSFMEYCVubfeArCsGPGDwb2yk=
2123
github.com/arduino/go-timeutils v0.0.0-20171220113728-d1dd9e313b1b h1:9hDi4F2st6dbLC3y4i02zFT5quS4X6iioWifGlVwfy4=
@@ -125,6 +127,8 @@ github.com/magiconair/properties v1.8.0 h1:LLgXmsheXeRoUOBOjtwPQCWIYqM/LU1ayDtDe
125127
github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
126128
github.com/magiconair/properties v1.8.1 h1:ZC2Vc7/ZFkGmsVC9KvOjumD+G5lXy2RtTKyzRKO2BQ4=
127129
github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
130+
github.com/marcinbor85/gohex v0.0.0-20200531163658-baab2527a9a2 h1:n7R8fUwWZUB2XtyzBNsYNNm9/XgOBj6pvLi7GLMCHtM=
131+
github.com/marcinbor85/gohex v0.0.0-20200531163658-baab2527a9a2/go.mod h1:Pb6XcsXyropB9LNHhnqaknG/vEwYztLkQzVCHv8sQ3M=
128132
github.com/mattn/go-colorable v0.1.2 h1:/bC9yWikZXAL9uJdulbSfyVNIR3n3trXl+v8+1sx8mU=
129133
github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
130134
github.com/mattn/go-isatty v0.0.8 h1:HLtExJ+uU2HOZ+wI0Tt5DtUDrx8yhUqDcp7fYERX4CE=

‎legacy/builder/merge_sketch_with_bootloader.go

Lines changed: 70 additions & 50 deletions
Original file line numberDiff line numberDiff line change
@@ -16,12 +16,15 @@
1616
package builder
1717

1818
import (
19-
"os"
19+
"math"
20+
"strconv"
2021
"strings"
2122

2223
"github.com/arduino/arduino-cli/legacy/builder/constants"
2324
"github.com/arduino/arduino-cli/legacy/builder/types"
25+
"github.com/arduino/arduino-cli/legacy/builder/utils"
2426
"github.com/arduino/go-paths-helper"
27+
"github.com/marcinbor85/gohex"
2528
"github.com/pkg/errors"
2629
)
2730

@@ -36,7 +39,6 @@ func (s *MergeSketchWithBootloader) Run(ctx *types.Context) error {
3639
buildPath := ctx.BuildPath
3740
sketch := ctx.Sketch
3841
sketchFileName := sketch.MainFile.Name.Base()
39-
logger := ctx.GetLogger()
4042

4143
sketchInBuildPath := buildPath.Join(sketchFileName + ".hex")
4244
sketchInSubfolder := buildPath.Join(constants.FOLDER_SKETCH, sketchFileName+".hex")
@@ -60,74 +62,92 @@ func (s *MergeSketchWithBootloader) Run(ctx *types.Context) error {
6062

6163
bootloaderPath := buildProperties.GetPath(constants.BUILD_PROPERTIES_RUNTIME_PLATFORM_PATH).Join(constants.FOLDER_BOOTLOADERS, bootloader)
6264
if bootloaderPath.NotExist() {
63-
logger.Fprintln(os.Stdout, constants.LOG_LEVEL_WARN, constants.MSG_BOOTLOADER_FILE_MISSING, bootloaderPath)
65+
utils.LogIfVerbose(constants.LOG_LEVEL_WARN, constants.MSG_BOOTLOADER_FILE_MISSING, bootloaderPath)
6466
return nil
6567
}
6668

6769
mergedSketchPath := builtSketchPath.Parent().Join(sketchFileName + ".with_bootloader.hex")
6870

69-
return merge(builtSketchPath, bootloaderPath, mergedSketchPath)
70-
}
71-
72-
func hexLineOnlyContainsFF(line string) bool {
73-
//:206FE000FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFB1
74-
if len(line) <= 11 {
75-
return false
71+
// Ignore merger errors for the first iteration
72+
maximumBinSize := 16000000
73+
if uploadMaxSize, ok := ctx.BuildProperties.GetOk(constants.PROPERTY_UPLOAD_MAX_SIZE); ok {
74+
maximumBinSize, _ = strconv.Atoi(uploadMaxSize)
75+
maximumBinSize *= 2
7676
}
77-
byteArray := []byte(line)
78-
for _, char := range byteArray[9:(len(byteArray) - 2)] {
79-
if char != 'F' {
80-
return false
81-
}
77+
err := merge(builtSketchPath, bootloaderPath, mergedSketchPath, maximumBinSize)
78+
if err != nil {
79+
utils.LogIfVerbose(constants.LOG_LEVEL_INFO, err.Error())
8280
}
83-
return true
84-
}
85-
86-
func extractActualBootloader(bootloader []string) []string {
8781

88-
var realBootloader []string
82+
return nil
83+
}
8984

90-
// skip until we find a line full of FFFFFF (except address and checksum)
91-
for i, row := range bootloader {
92-
if hexLineOnlyContainsFF(row) {
93-
realBootloader = bootloader[i:len(bootloader)]
94-
break
95-
}
85+
func merge(builtSketchPath, bootloaderPath, mergedSketchPath *paths.Path, maximumBinSize int) error {
86+
if bootloaderPath.Ext() == ".bin" {
87+
bootloaderPath = paths.New(strings.TrimSuffix(bootloaderPath.String(), ".bin") + ".hex")
9688
}
9789

98-
// drop all "empty" lines
99-
for i, row := range realBootloader {
100-
if !hexLineOnlyContainsFF(row) {
101-
realBootloader = realBootloader[i:len(realBootloader)]
102-
break
90+
memBoot := gohex.NewMemory()
91+
if bootFile, err := bootloaderPath.Open(); err == nil {
92+
defer bootFile.Close()
93+
if err := memBoot.ParseIntelHex(bootFile); err != nil {
94+
return errors.New(bootFile.Name() + " " + err.Error())
10395
}
96+
} else {
97+
return err
10498
}
10599

106-
if len(realBootloader) == 0 {
107-
// we didn't find any line full of FFFF, thus it's a standalone bootloader
108-
realBootloader = bootloader
100+
memSketch := gohex.NewMemory()
101+
if buildFile, err := builtSketchPath.Open(); err == nil {
102+
defer buildFile.Close()
103+
if err := memSketch.ParseIntelHex(buildFile); err != nil {
104+
return errors.New(buildFile.Name() + " " + err.Error())
105+
}
106+
} else {
107+
return err
109108
}
110109

111-
return realBootloader
112-
}
110+
memMerged := gohex.NewMemory()
111+
initialAddress := uint32(math.MaxUint32)
112+
lastAddress := uint32(0)
113113

114-
func merge(builtSketchPath, bootloaderPath, mergedSketchPath *paths.Path) error {
115-
sketch, err := builtSketchPath.ReadFileAsLines()
116-
if err != nil {
117-
return errors.WithStack(err)
114+
for _, segment := range memBoot.GetDataSegments() {
115+
if err := memMerged.AddBinary(segment.Address, segment.Data); err != nil {
116+
continue
117+
}
118+
if segment.Address < initialAddress {
119+
initialAddress = segment.Address
120+
}
121+
if segment.Address+uint32(len(segment.Data)) > lastAddress {
122+
lastAddress = segment.Address + uint32(len(segment.Data))
123+
}
118124
}
119-
sketch = sketch[:len(sketch)-2]
120-
121-
bootloader, err := bootloaderPath.ReadFileAsLines()
122-
if err != nil {
123-
return errors.WithStack(err)
125+
for _, segment := range memSketch.GetDataSegments() {
126+
if err := memMerged.AddBinary(segment.Address, segment.Data); err != nil {
127+
continue
128+
}
129+
if segment.Address < initialAddress {
130+
initialAddress = segment.Address
131+
}
132+
if segment.Address+uint32(len(segment.Data)) > lastAddress {
133+
lastAddress = segment.Address + uint32(len(segment.Data))
134+
}
124135
}
125136

126-
realBootloader := extractActualBootloader(bootloader)
127-
128-
for _, row := range realBootloader {
129-
sketch = append(sketch, row)
137+
if mergeFile, err := mergedSketchPath.Create(); err == nil {
138+
defer mergeFile.Close()
139+
memMerged.DumpIntelHex(mergeFile, 16)
140+
} else {
141+
return err
130142
}
131143

132-
return mergedSketchPath.WriteFile([]byte(strings.Join(sketch, "\n")))
144+
// Write out a .bin if the addresses doesn't go too far away from origin
145+
// (and consequently produce a very large bin)
146+
size := lastAddress - initialAddress
147+
if size > uint32(maximumBinSize) {
148+
return nil
149+
}
150+
mergedSketchPathBin := paths.New(strings.TrimSuffix(mergedSketchPath.String(), ".hex") + ".bin")
151+
data := memMerged.ToBinary(initialAddress, size, 0xFF)
152+
return mergedSketchPathBin.WriteFile(data)
133153
}

‎legacy/builder/test/merge_sketch_with_bootloader_test.go

Lines changed: 98 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616
package test
1717

1818
import (
19+
"fmt"
1920
"path/filepath"
2021
"strings"
2122
"testing"
@@ -46,8 +47,36 @@ func TestMergeSketchWithBootloader(t *testing.T) {
4647
err := buildPath.Join("sketch").MkdirAll()
4748
NoError(t, err)
4849

49-
fakeSketchHex := "row 1\n" +
50-
"row 2\n"
50+
fakeSketchHex := `:100000000C9434000C9446000C9446000C9446006A
51+
:100010000C9446000C9446000C9446000C94460048
52+
:100020000C9446000C9446000C9446000C94460038
53+
:100030000C9446000C9446000C9446000C94460028
54+
:100040000C9448000C9446000C9446000C94460016
55+
:100050000C9446000C9446000C9446000C94460008
56+
:100060000C9446000C94460011241FBECFEFD8E03C
57+
:10007000DEBFCDBF21E0A0E0B1E001C01D92A930FC
58+
:10008000B207E1F70E9492000C94DC000C9400008F
59+
:100090001F920F920FB60F9211242F933F938F93BD
60+
:1000A0009F93AF93BF938091050190910601A0911A
61+
:1000B0000701B09108013091040123E0230F2D378F
62+
:1000C00020F40196A11DB11D05C026E8230F02965C
63+
:1000D000A11DB11D20930401809305019093060199
64+
:1000E000A0930701B0930801809100019091010154
65+
:1000F000A0910201B09103010196A11DB11D809351
66+
:10010000000190930101A0930201B0930301BF91FC
67+
:10011000AF919F918F913F912F910F900FBE0F90B4
68+
:100120001F901895789484B5826084BD84B58160F1
69+
:1001300084BD85B5826085BD85B5816085BD8091B2
70+
:100140006E00816080936E0010928100809181002A
71+
:100150008260809381008091810081608093810022
72+
:10016000809180008160809380008091B1008460E4
73+
:100170008093B1008091B00081608093B000809145
74+
:100180007A00846080937A0080917A008260809304
75+
:100190007A0080917A00816080937A0080917A0061
76+
:1001A000806880937A001092C100C0E0D0E0209770
77+
:0C01B000F1F30E940000FBCFF894FFCF99
78+
:00000001FF
79+
`
5180
err = buildPath.Join("sketch", "sketch.ino.hex").WriteFile([]byte(fakeSketchHex))
5281
NoError(t, err)
5382

@@ -65,8 +94,8 @@ func TestMergeSketchWithBootloader(t *testing.T) {
6594
NoError(t, err)
6695
mergedSketchHex := string(bytes)
6796

68-
require.True(t, strings.HasPrefix(mergedSketchHex, "row 1\n:107E0000112484B714BE81FFF0D085E080938100F7\n"))
69-
require.True(t, strings.HasSuffix(mergedSketchHex, ":0400000300007E007B\n:00000001FF\n"))
97+
require.Contains(t, mergedSketchHex, ":100000000C9434000C9446000C9446000C9446006A\n")
98+
require.True(t, strings.HasSuffix(mergedSketchHex, ":00000001FF\n"))
7099
}
71100

72101
func TestMergeSketchWithBootloaderSketchInBuildPath(t *testing.T) {
@@ -88,8 +117,36 @@ func TestMergeSketchWithBootloaderSketchInBuildPath(t *testing.T) {
88117
err := buildPath.Join("sketch").MkdirAll()
89118
NoError(t, err)
90119

91-
fakeSketchHex := "row 1\n" +
92-
"row 2\n"
120+
fakeSketchHex := `:100000000C9434000C9446000C9446000C9446006A
121+
:100010000C9446000C9446000C9446000C94460048
122+
:100020000C9446000C9446000C9446000C94460038
123+
:100030000C9446000C9446000C9446000C94460028
124+
:100040000C9448000C9446000C9446000C94460016
125+
:100050000C9446000C9446000C9446000C94460008
126+
:100060000C9446000C94460011241FBECFEFD8E03C
127+
:10007000DEBFCDBF21E0A0E0B1E001C01D92A930FC
128+
:10008000B207E1F70E9492000C94DC000C9400008F
129+
:100090001F920F920FB60F9211242F933F938F93BD
130+
:1000A0009F93AF93BF938091050190910601A0911A
131+
:1000B0000701B09108013091040123E0230F2D378F
132+
:1000C00020F40196A11DB11D05C026E8230F02965C
133+
:1000D000A11DB11D20930401809305019093060199
134+
:1000E000A0930701B0930801809100019091010154
135+
:1000F000A0910201B09103010196A11DB11D809351
136+
:10010000000190930101A0930201B0930301BF91FC
137+
:10011000AF919F918F913F912F910F900FBE0F90B4
138+
:100120001F901895789484B5826084BD84B58160F1
139+
:1001300084BD85B5826085BD85B5816085BD8091B2
140+
:100140006E00816080936E0010928100809181002A
141+
:100150008260809381008091810081608093810022
142+
:10016000809180008160809380008091B1008460E4
143+
:100170008093B1008091B00081608093B000809145
144+
:100180007A00846080937A0080917A008260809304
145+
:100190007A0080917A00816080937A0080917A0061
146+
:1001A000806880937A001092C100C0E0D0E0209770
147+
:0C01B000F1F30E940000FBCFF894FFCF99
148+
:00000001FF
149+
`
93150
err = buildPath.Join("sketch.ino.hex").WriteFile([]byte(fakeSketchHex))
94151
NoError(t, err)
95152

@@ -107,8 +164,9 @@ func TestMergeSketchWithBootloaderSketchInBuildPath(t *testing.T) {
107164
NoError(t, err)
108165
mergedSketchHex := string(bytes)
109166

110-
require.True(t, strings.HasPrefix(mergedSketchHex, "row 1\n:107E0000112484B714BE81FFF0D085E080938100F7\n"))
111-
require.True(t, strings.HasSuffix(mergedSketchHex, ":0400000300007E007B\n:00000001FF\n"))
167+
fmt.Println(string(mergedSketchHex))
168+
require.Contains(t, mergedSketchHex, ":100000000C9434000C9446000C9446000C9446006A\n")
169+
require.True(t, strings.HasSuffix(mergedSketchHex, ":00000001FF\n"))
112170
}
113171

114172
func TestMergeSketchWithBootloaderWhenNoBootloaderAvailable(t *testing.T) {
@@ -168,8 +226,36 @@ func TestMergeSketchWithBootloaderPathIsParameterized(t *testing.T) {
168226
err := buildPath.Join("sketch").MkdirAll()
169227
NoError(t, err)
170228

171-
fakeSketchHex := "row 1\n" +
172-
"row 2\n"
229+
fakeSketchHex := `:100000000C9434000C9446000C9446000C9446006A
230+
:100010000C9446000C9446000C9446000C94460048
231+
:100020000C9446000C9446000C9446000C94460038
232+
:100030000C9446000C9446000C9446000C94460028
233+
:100040000C9448000C9446000C9446000C94460016
234+
:100050000C9446000C9446000C9446000C94460008
235+
:100060000C9446000C94460011241FBECFEFD8E03C
236+
:10007000DEBFCDBF21E0A0E0B1E001C01D92A930FC
237+
:10008000B207E1F70E9492000C94DC000C9400008F
238+
:100090001F920F920FB60F9211242F933F938F93BD
239+
:1000A0009F93AF93BF938091050190910601A0911A
240+
:1000B0000701B09108013091040123E0230F2D378F
241+
:1000C00020F40196A11DB11D05C026E8230F02965C
242+
:1000D000A11DB11D20930401809305019093060199
243+
:1000E000A0930701B0930801809100019091010154
244+
:1000F000A0910201B09103010196A11DB11D809351
245+
:10010000000190930101A0930201B0930301BF91FC
246+
:10011000AF919F918F913F912F910F900FBE0F90B4
247+
:100120001F901895789484B5826084BD84B58160F1
248+
:1001300084BD85B5826085BD85B5816085BD8091B2
249+
:100140006E00816080936E0010928100809181002A
250+
:100150008260809381008091810081608093810022
251+
:10016000809180008160809380008091B1008460E4
252+
:100170008093B1008091B00081608093B000809145
253+
:100180007A00846080937A0080917A008260809304
254+
:100190007A0080917A00816080937A0080917A0061
255+
:1001A000806880937A001092C100C0E0D0E0209770
256+
:0C01B000F1F30E940000FBCFF894FFCF99
257+
:00000001FF
258+
`
173259
err = buildPath.Join("sketch", "sketch.ino.hex").WriteFile([]byte(fakeSketchHex))
174260
NoError(t, err)
175261

@@ -187,6 +273,6 @@ func TestMergeSketchWithBootloaderPathIsParameterized(t *testing.T) {
187273
NoError(t, err)
188274
mergedSketchHex := string(bytes)
189275

190-
require.True(t, strings.HasPrefix(mergedSketchHex, "row 1\n:020000023000CC"))
191-
require.True(t, strings.HasSuffix(mergedSketchHex, ":040000033000E000E9\n:00000001FF\n"))
276+
require.Contains(t, mergedSketchHex, ":100000000C9434000C9446000C9446000C9446006A\n")
277+
require.True(t, strings.HasSuffix(mergedSketchHex, ":00000001FF\n"))
192278
}

0 commit comments

Comments
 (0)
pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy