Skip to content

Improve hex merger #295

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 5 commits into from
Closed

Improve hex merger #295

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
1707aec
Start introducing a proper hex merger
facchinm Sep 14, 2018
7a6bad6
Properly handle hex<->bin operations
facchinm Sep 20, 2018
9a61ec1
Fix merge bootloader test
facchinm Sep 20, 2018
d6536f4
Travis: add dependency from gohex
facchinm Sep 20, 2018
d158e15
Limit _with_bootloader bin file to 16MB or 2*max_upload_size
facchinm Mar 7, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .travis.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ install:
- go get github.com/wadey/gocovmerge
- go get github.com/arduino/go-properties-map
- go get github.com/arduino/go-timeutils
- go get github.com/marcinbor85/gohex

script:
- go get github.com/arduino/arduino-builder/arduino-builder
Expand Down
127 changes: 78 additions & 49 deletions merge_sketch_with_bootloader.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,13 +30,17 @@
package builder

import (
"github.com/arduino/arduino-builder/constants"
"github.com/arduino/arduino-builder/i18n"
"github.com/arduino/arduino-builder/types"
"github.com/arduino/arduino-builder/utils"
"errors"
"math"
"os"
"path/filepath"
"strings"
"strconv"

"github.com/arduino/arduino-builder/constants"
"github.com/arduino/arduino-builder/types"
"github.com/arduino/arduino-builder/utils"
"github.com/marcinbor85/gohex"
)

type MergeSketchWithBootloader struct{}
Expand Down Expand Up @@ -80,70 +84,95 @@ func (s *MergeSketchWithBootloader) Run(ctx *types.Context) error {

mergedSketchPath := filepath.Join(filepath.Dir(builtSketchPath), sketchFileName+".with_bootloader.hex")

err := merge(builtSketchPath, bootloaderPath, mergedSketchPath)

return err
}

func hexLineOnlyContainsFF(line string) bool {
//:206FE000FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFB1
if len(line) <= 11 {
return false
// Ignore merger errors for the first iteration
maximumBinSize := 16000000
if uploadMaxSize, ok := ctx.BuildProperties[constants.PROPERTY_UPLOAD_MAX_SIZE]; ok {
maximumBinSize, _ = strconv.Atoi(uploadMaxSize)
maximumBinSize *= 2
}
byteArray := []byte(line)
for _, char := range byteArray[9:(len(byteArray) - 2)] {
if char != 'F' {
return false
}
err := merge(builtSketchPath, bootloaderPath, mergedSketchPath, maximumBinSize)
if err != nil {
logger.Fprintln(os.Stdout, constants.LOG_LEVEL_WARN, err.Error())
}
return true
}

func extractActualBootloader(bootloader []string) []string {
return nil
}

var realBootloader []string
func merge(builtSketchPath, bootloaderPath, mergedSketchPath string, maximumBinSize int) error {

// skip until we find a line full of FFFFFF (except address and checksum)
for i, row := range bootloader {
if hexLineOnlyContainsFF(row) {
realBootloader = bootloader[i:len(bootloader)]
break
}
if filepath.Ext(bootloaderPath) == ".bin" {
bootloaderPath = strings.TrimSuffix(bootloaderPath, ".bin") + ".hex"
}

// drop all "empty" lines
for i, row := range realBootloader {
if !hexLineOnlyContainsFF(row) {
realBootloader = realBootloader[i:len(realBootloader)]
break
}
bootFile, err := os.Open(bootloaderPath)
if err != nil {
return err
}
defer bootFile.Close()

if len(realBootloader) == 0 {
// we didn't find any line full of FFFF, thus it's a standalone bootloader
realBootloader = bootloader
mem_boot := gohex.NewMemory()
err = mem_boot.ParseIntelHex(bootFile)
if err != nil {
return errors.New(bootFile.Name() + " " + err.Error())
}

return realBootloader
}
buildFile, err := os.Open(builtSketchPath)
if err != nil {
return err
}
defer buildFile.Close()

func merge(builtSketchPath, bootloaderPath, mergedSketchPath string) error {
sketch, err := utils.ReadFileToRows(builtSketchPath)
mem_sketch := gohex.NewMemory()
err = mem_sketch.ParseIntelHex(buildFile)
if err != nil {
return i18n.WrapError(err)
return errors.New(buildFile.Name() + " " + err.Error())
}
sketch = sketch[:len(sketch)-2]

bootloader, err := utils.ReadFileToRows(bootloaderPath)
mem_merge := gohex.NewMemory()
initial_address := uint32(math.MaxUint32)
last_address := uint32(0)

for _, segment := range mem_boot.GetDataSegments() {
err = mem_merge.AddBinary(segment.Address, segment.Data)
if err != nil {
continue
} else {
if segment.Address < initial_address {
initial_address = segment.Address
}
if segment.Address+uint32(len(segment.Data)) > last_address {
last_address = segment.Address + uint32(len(segment.Data))
}
}
}
for _, segment := range mem_sketch.GetDataSegments() {
err = mem_merge.AddBinary(segment.Address, segment.Data)
if err != nil {
continue
}
if segment.Address < initial_address {
initial_address = segment.Address
}
if segment.Address+uint32(len(segment.Data)) > last_address {
last_address = segment.Address + uint32(len(segment.Data))
}
}

mergeFile, err := os.Create(mergedSketchPath)
if err != nil {
return i18n.WrapError(err)
return err
}
defer mergeFile.Close()

mem_merge.DumpIntelHex(mergeFile, 16)

realBootloader := extractActualBootloader(bootloader)
mergedSketchPathBin := strings.TrimSuffix(mergedSketchPath, ".hex") + ".bin"

for _, row := range realBootloader {
sketch = append(sketch, row)
size := last_address - initial_address
if (size > uint32(maximumBinSize)) {
return nil
}

return utils.WriteFile(mergedSketchPath, strings.Join(sketch, "\n"))
bytes := mem_merge.ToBinary(initial_address, last_address-initial_address, 0xFF)
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This must only be executed if address space is continuous (otherwise we can get a huge file as result)

return utils.WriteFile(mergedSketchPathBin, string(bytes))
}
116 changes: 99 additions & 17 deletions test/merge_sketch_with_bootloader_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,16 +30,17 @@
package test

import (
"github.com/arduino/arduino-builder"
"github.com/arduino/arduino-builder/constants"
"github.com/arduino/arduino-builder/types"
"github.com/arduino/arduino-builder/utils"
"github.com/stretchr/testify/require"
"io/ioutil"
"os"
"path/filepath"
"strings"
"testing"

"github.com/arduino/arduino-builder"
"github.com/arduino/arduino-builder/constants"
"github.com/arduino/arduino-builder/types"
"github.com/arduino/arduino-builder/utils"
"github.com/stretchr/testify/require"
)

func TestMergeSketchWithBootloader(t *testing.T) {
Expand All @@ -61,8 +62,35 @@ func TestMergeSketchWithBootloader(t *testing.T) {
err := utils.EnsureFolderExists(filepath.Join(buildPath, "sketch"))
NoError(t, err)

fakeSketchHex := "row 1\n" +
"row 2\n"
fakeSketchHex := ":100000000C9434000C9446000C9446000C9446006A\n" +
":100010000C9446000C9446000C9446000C94460048\n" +
":100020000C9446000C9446000C9446000C94460038\n" +
":100030000C9446000C9446000C9446000C94460028\n" +
":100040000C9448000C9446000C9446000C94460016\n" +
":100050000C9446000C9446000C9446000C94460008\n" +
":100060000C9446000C94460011241FBECFEFD8E03C\n" +
":10007000DEBFCDBF21E0A0E0B1E001C01D92A930FC\n" +
":10008000B207E1F70E9492000C94DC000C9400008F\n" +
":100090001F920F920FB60F9211242F933F938F93BD\n" +
":1000A0009F93AF93BF938091050190910601A0911A\n" +
":1000B0000701B09108013091040123E0230F2D378F\n" +
":1000C00020F40196A11DB11D05C026E8230F02965C\n" +
":1000D000A11DB11D20930401809305019093060199\n" +
":1000E000A0930701B0930801809100019091010154\n" +
":1000F000A0910201B09103010196A11DB11D809351\n" +
":10010000000190930101A0930201B0930301BF91FC\n" +
":10011000AF919F918F913F912F910F900FBE0F90B4\n" +
":100120001F901895789484B5826084BD84B58160F1\n" +
":1001300084BD85B5826085BD85B5816085BD8091B2\n" +
":100140006E00816080936E0010928100809181002A\n" +
":100150008260809381008091810081608093810022\n" +
":10016000809180008160809380008091B1008460E4\n" +
":100170008093B1008091B00081608093B000809145\n" +
":100180007A00846080937A0080917A008260809304\n" +
":100190007A0080917A00816080937A0080917A0061\n" +
":1001A000806880937A001092C100C0E0D0E0209770\n" +
":0C01B000F1F30E940000FBCFF894FFCF99\n" +
":00000001FF\n"
err = utils.WriteFile(filepath.Join(buildPath, "sketch", "sketch.ino.hex"), fakeSketchHex)
NoError(t, err)

Expand All @@ -80,8 +108,8 @@ func TestMergeSketchWithBootloader(t *testing.T) {
NoError(t, err)
mergedSketchHex := string(bytes)

require.True(t, strings.HasPrefix(mergedSketchHex, "row 1\n:107E0000112484B714BE81FFF0D085E080938100F7\n"))
require.True(t, strings.HasSuffix(mergedSketchHex, ":0400000300007E007B\n:00000001FF\n"))
require.True(t, strings.HasPrefix(mergedSketchHex, ":100000000C9434000C9446000C9446000C9446006A\n"))
require.True(t, strings.HasSuffix(mergedSketchHex, ":00000001FF\n"))
}

func TestMergeSketchWithBootloaderSketchInBuildPath(t *testing.T) {
Expand All @@ -103,8 +131,35 @@ func TestMergeSketchWithBootloaderSketchInBuildPath(t *testing.T) {
err := utils.EnsureFolderExists(filepath.Join(buildPath, "sketch"))
NoError(t, err)

fakeSketchHex := "row 1\n" +
"row 2\n"
fakeSketchHex := ":100000000C9434000C9446000C9446000C9446006A\n" +
":100010000C9446000C9446000C9446000C94460048\n" +
":100020000C9446000C9446000C9446000C94460038\n" +
":100030000C9446000C9446000C9446000C94460028\n" +
":100040000C9448000C9446000C9446000C94460016\n" +
":100050000C9446000C9446000C9446000C94460008\n" +
":100060000C9446000C94460011241FBECFEFD8E03C\n" +
":10007000DEBFCDBF21E0A0E0B1E001C01D92A930FC\n" +
":10008000B207E1F70E9492000C94DC000C9400008F\n" +
":100090001F920F920FB60F9211242F933F938F93BD\n" +
":1000A0009F93AF93BF938091050190910601A0911A\n" +
":1000B0000701B09108013091040123E0230F2D378F\n" +
":1000C00020F40196A11DB11D05C026E8230F02965C\n" +
":1000D000A11DB11D20930401809305019093060199\n" +
":1000E000A0930701B0930801809100019091010154\n" +
":1000F000A0910201B09103010196A11DB11D809351\n" +
":10010000000190930101A0930201B0930301BF91FC\n" +
":10011000AF919F918F913F912F910F900FBE0F90B4\n" +
":100120001F901895789484B5826084BD84B58160F1\n" +
":1001300084BD85B5826085BD85B5816085BD8091B2\n" +
":100140006E00816080936E0010928100809181002A\n" +
":100150008260809381008091810081608093810022\n" +
":10016000809180008160809380008091B1008460E4\n" +
":100170008093B1008091B00081608093B000809145\n" +
":100180007A00846080937A0080917A008260809304\n" +
":100190007A0080917A00816080937A0080917A0061\n" +
":1001A000806880937A001092C100C0E0D0E0209770\n" +
":0C01B000F1F30E940000FBCFF894FFCF99\n" +
":00000001FF\n"
err = utils.WriteFile(filepath.Join(buildPath, "sketch.ino.hex"), fakeSketchHex)
NoError(t, err)

Expand All @@ -122,8 +177,8 @@ func TestMergeSketchWithBootloaderSketchInBuildPath(t *testing.T) {
NoError(t, err)
mergedSketchHex := string(bytes)

require.True(t, strings.HasPrefix(mergedSketchHex, "row 1\n:107E0000112484B714BE81FFF0D085E080938100F7\n"))
require.True(t, strings.HasSuffix(mergedSketchHex, ":0400000300007E007B\n:00000001FF\n"))
require.True(t, strings.HasPrefix(mergedSketchHex, ":100000000C9434000C9446000C9446000C9446006A\n"))
require.True(t, strings.HasSuffix(mergedSketchHex, ":00000001FF\n"))
}

func TestMergeSketchWithBootloaderWhenNoBootloaderAvailable(t *testing.T) {
Expand Down Expand Up @@ -183,8 +238,35 @@ func TestMergeSketchWithBootloaderPathIsParameterized(t *testing.T) {
err := utils.EnsureFolderExists(filepath.Join(buildPath, "sketch"))
NoError(t, err)

fakeSketchHex := "row 1\n" +
"row 2\n"
fakeSketchHex := ":100000000C9434000C9446000C9446000C9446006A\n" +
":100010000C9446000C9446000C9446000C94460048\n" +
":100020000C9446000C9446000C9446000C94460038\n" +
":100030000C9446000C9446000C9446000C94460028\n" +
":100040000C9448000C9446000C9446000C94460016\n" +
":100050000C9446000C9446000C9446000C94460008\n" +
":100060000C9446000C94460011241FBECFEFD8E03C\n" +
":10007000DEBFCDBF21E0A0E0B1E001C01D92A930FC\n" +
":10008000B207E1F70E9492000C94DC000C9400008F\n" +
":100090001F920F920FB60F9211242F933F938F93BD\n" +
":1000A0009F93AF93BF938091050190910601A0911A\n" +
":1000B0000701B09108013091040123E0230F2D378F\n" +
":1000C00020F40196A11DB11D05C026E8230F02965C\n" +
":1000D000A11DB11D20930401809305019093060199\n" +
":1000E000A0930701B0930801809100019091010154\n" +
":1000F000A0910201B09103010196A11DB11D809351\n" +
":10010000000190930101A0930201B0930301BF91FC\n" +
":10011000AF919F918F913F912F910F900FBE0F90B4\n" +
":100120001F901895789484B5826084BD84B58160F1\n" +
":1001300084BD85B5826085BD85B5816085BD8091B2\n" +
":100140006E00816080936E0010928100809181002A\n" +
":100150008260809381008091810081608093810022\n" +
":10016000809180008160809380008091B1008460E4\n" +
":100170008093B1008091B00081608093B000809145\n" +
":100180007A00846080937A0080917A008260809304\n" +
":100190007A0080917A00816080937A0080917A0061\n" +
":1001A000806880937A001092C100C0E0D0E0209770\n" +
":0C01B000F1F30E940000FBCFF894FFCF99\n" +
":00000001FF\n"
err = utils.WriteFile(filepath.Join(buildPath, "sketch", "sketch.ino.hex"), fakeSketchHex)
NoError(t, err)

Expand All @@ -202,6 +284,6 @@ func TestMergeSketchWithBootloaderPathIsParameterized(t *testing.T) {
NoError(t, err)
mergedSketchHex := string(bytes)

require.True(t, strings.HasPrefix(mergedSketchHex, "row 1\n:020000023000CC"))
require.True(t, strings.HasSuffix(mergedSketchHex, ":040000033000E000E9\n:00000001FF\n"))
require.True(t, strings.HasPrefix(mergedSketchHex, ":100000000C9434000C9446000C9446000C9446006A\n"))
require.True(t, strings.HasSuffix(mergedSketchHex, ":00000001FF\n"))
}