Skip to content

Commit 3f3782f

Browse files
randall77randall77
committed
cmd/compile: allow all of the preamble to be preemptible
We currently make some parts of the preamble unpreemptible because it confuses morestack. See comments in the code. Instead, have morestack handle those weird cases so we can remove unpreemptible marks from most places. This CL makes user functions preemptible everywhere if they have no write barriers (at least, on x86). In cmd/go the fraction of functions that need preemptible markings drops from 82% to 36%. Makes the cmd/go binary 0.3% smaller. Update #35470 Change-Id: Ic83d5eabfd0f6d239a92e65684bcce7e67ff30bb Reviewed-on: https://go-review.googlesource.com/c/go/+/648518 Auto-Submit: Keith Randall <[email protected]> Reviewed-by: Keith Randall <[email protected]> LUCI-TryBot-Result: Go LUCI <[email protected]> Reviewed-by: Cherry Mui <[email protected]>
1 parent dc1e255 commit 3f3782f

File tree

12 files changed

+58
-126
lines changed

12 files changed

+58
-126
lines changed

‎src/cmd/internal/obj/arm/obj5.go

+2-13
Original file line numberDiff line numberDiff line change
@@ -703,12 +703,6 @@ func (c *ctxt5) stacksplit(p *obj.Prog, framesize int32) *obj.Prog {
703703
p.To.Type = obj.TYPE_REG
704704
p.To.Reg = REG_R1
705705

706-
// Mark the stack bound check and morestack call async nonpreemptible.
707-
// If we get preempted here, when resumed the preemption request is
708-
// cleared, but we'll still call morestack, which will double the stack
709-
// unnecessarily. See issue #35470.
710-
p = c.ctxt.StartUnsafePoint(p, c.newprog)
711-
712706
if framesize <= abi.StackSmall {
713707
// small stack: SP < stackguard
714708
// CMP stackguard, SP
@@ -772,8 +766,6 @@ func (c *ctxt5) stacksplit(p *obj.Prog, framesize int32) *obj.Prog {
772766
bls.As = ABLS
773767
bls.To.Type = obj.TYPE_BRANCH
774768

775-
end := c.ctxt.EndUnsafePoint(bls, c.newprog, -1)
776-
777769
var last *obj.Prog
778770
for last = c.cursym.Func().Text; last.Link != nil; last = last.Link {
779771
}
@@ -786,7 +778,6 @@ func (c *ctxt5) stacksplit(p *obj.Prog, framesize int32) *obj.Prog {
786778
spfix.Spadj = -framesize
787779

788780
pcdata := c.ctxt.EmitEntryStackMap(c.cursym, spfix, c.newprog)
789-
pcdata = c.ctxt.StartUnsafePoint(pcdata, c.newprog)
790781

791782
// MOVW LR, R3
792783
movw := obj.Appendp(pcdata, c.newprog)
@@ -811,16 +802,14 @@ func (c *ctxt5) stacksplit(p *obj.Prog, framesize int32) *obj.Prog {
811802
}
812803
call.To.Sym = c.ctxt.Lookup(morestack)
813804

814-
pcdata = c.ctxt.EndUnsafePoint(call, c.newprog, -1)
815-
816805
// B start
817-
b := obj.Appendp(pcdata, c.newprog)
806+
b := obj.Appendp(call, c.newprog)
818807
b.As = obj.AJMP
819808
b.To.Type = obj.TYPE_BRANCH
820809
b.To.SetTarget(startPred.Link)
821810
b.Spadj = +framesize
822811

823-
return end
812+
return bls
824813
}
825814

826815
var unaryDst = map[obj.As]bool{

‎src/cmd/internal/obj/arm64/obj7.go

+2-13
Original file line numberDiff line numberDiff line change
@@ -163,12 +163,6 @@ func (c *ctxt7) stacksplit(p *obj.Prog, framesize int32) *obj.Prog {
163163
p.To.Type = obj.TYPE_REG
164164
p.To.Reg = REGRT1
165165

166-
// Mark the stack bound check and morestack call async nonpreemptible.
167-
// If we get preempted here, when resumed the preemption request is
168-
// cleared, but we'll still call morestack, which will double the stack
169-
// unnecessarily. See issue #35470.
170-
p = c.ctxt.StartUnsafePoint(p, c.newprog)
171-
172166
q := (*obj.Prog)(nil)
173167
if framesize <= abi.StackSmall {
174168
// small stack: SP < stackguard
@@ -235,8 +229,6 @@ func (c *ctxt7) stacksplit(p *obj.Prog, framesize int32) *obj.Prog {
235229
bls.As = ABLS
236230
bls.To.Type = obj.TYPE_BRANCH
237231

238-
end := c.ctxt.EndUnsafePoint(bls, c.newprog, -1)
239-
240232
var last *obj.Prog
241233
for last = c.cursym.Func().Text; last.Link != nil; last = last.Link {
242234
}
@@ -249,7 +241,6 @@ func (c *ctxt7) stacksplit(p *obj.Prog, framesize int32) *obj.Prog {
249241
spfix.Spadj = -framesize
250242

251243
pcdata := c.ctxt.EmitEntryStackMap(c.cursym, spfix, c.newprog)
252-
pcdata = c.ctxt.StartUnsafePoint(pcdata, c.newprog)
253244

254245
if q != nil {
255246
q.To.SetTarget(pcdata)
@@ -289,9 +280,7 @@ func (c *ctxt7) stacksplit(p *obj.Prog, framesize int32) *obj.Prog {
289280
}
290281
call.To.Sym = c.ctxt.Lookup(morestack)
291282

292-
// The instructions which unspill regs should be preemptible.
293-
pcdata = c.ctxt.EndUnsafePoint(call, c.newprog, -1)
294-
unspill := c.cursym.Func().UnspillRegisterArgs(pcdata, c.newprog)
283+
unspill := c.cursym.Func().UnspillRegisterArgs(call, c.newprog)
295284

296285
// B start
297286
jmp := obj.Appendp(unspill, c.newprog)
@@ -300,7 +289,7 @@ func (c *ctxt7) stacksplit(p *obj.Prog, framesize int32) *obj.Prog {
300289
jmp.To.SetTarget(startPred.Link)
301290
jmp.Spadj = +framesize
302291

303-
return end
292+
return bls
304293
}
305294

306295
func progedit(ctxt *obj.Link, p *obj.Prog, newprog obj.ProgAlloc) {

‎src/cmd/internal/obj/loong64/obj.go

+1-10
Original file line numberDiff line numberDiff line change
@@ -717,12 +717,6 @@ func (c *ctxt0) stacksplit(p *obj.Prog, framesize int32) *obj.Prog {
717717
p.To.Type = obj.TYPE_REG
718718
p.To.Reg = REG_R20
719719

720-
// Mark the stack bound check and morestack call async nonpreemptible.
721-
// If we get preempted here, when resumed the preemption request is
722-
// cleared, but we'll still call morestack, which will double the stack
723-
// unnecessarily. See issue #35470.
724-
p = c.ctxt.StartUnsafePoint(p, c.newprog)
725-
726720
var q *obj.Prog
727721
if framesize <= abi.StackSmall {
728722
// small stack: SP < stackguard
@@ -794,7 +788,7 @@ func (c *ctxt0) stacksplit(p *obj.Prog, framesize int32) *obj.Prog {
794788
p.To.Type = obj.TYPE_BRANCH
795789
p.Mark |= BRANCH
796790

797-
end := c.ctxt.EndUnsafePoint(p, c.newprog, -1)
791+
end := p
798792

799793
var last *obj.Prog
800794
for last = c.cursym.Func().Text; last.Link != nil; last = last.Link {
@@ -808,7 +802,6 @@ func (c *ctxt0) stacksplit(p *obj.Prog, framesize int32) *obj.Prog {
808802
spfix.Spadj = -framesize
809803

810804
pcdata := c.ctxt.EmitEntryStackMap(c.cursym, spfix, c.newprog)
811-
pcdata = c.ctxt.StartUnsafePoint(pcdata, c.newprog)
812805

813806
if q != nil {
814807
q.To.SetTarget(pcdata)
@@ -843,8 +836,6 @@ func (c *ctxt0) stacksplit(p *obj.Prog, framesize int32) *obj.Prog {
843836
}
844837
call.Mark |= BRANCH
845838

846-
// The instructions which unspill regs should be preemptible.
847-
pcdata = c.ctxt.EndUnsafePoint(call, c.newprog, -1)
848839
unspill := c.cursym.Func().UnspillRegisterArgs(pcdata, c.newprog)
849840

850841
// JMP start

‎src/cmd/internal/obj/mips/obj0.go

-8
Original file line numberDiff line numberDiff line change
@@ -767,12 +767,6 @@ func (c *ctxt0) stacksplit(p *obj.Prog, framesize int32) *obj.Prog {
767767
p.To.Type = obj.TYPE_REG
768768
p.To.Reg = REG_R1
769769

770-
// Mark the stack bound check and morestack call async nonpreemptible.
771-
// If we get preempted here, when resumed the preemption request is
772-
// cleared, but we'll still call morestack, which will double the stack
773-
// unnecessarily. See issue #35470.
774-
p = c.ctxt.StartUnsafePoint(p, c.newprog)
775-
776770
var q *obj.Prog
777771
if framesize <= abi.StackSmall {
778772
// small stack: SP < stackguard
@@ -876,8 +870,6 @@ func (c *ctxt0) stacksplit(p *obj.Prog, framesize int32) *obj.Prog {
876870
}
877871
p.Mark |= BRANCH
878872

879-
p = c.ctxt.EndUnsafePoint(p, c.newprog, -1)
880-
881873
// JMP start
882874
p = obj.Appendp(p, c.newprog)
883875

‎src/cmd/internal/obj/plist.go

+2-23
Original file line numberDiff line numberDiff line change
@@ -262,16 +262,8 @@ func (ctxt *Link) GloblPos(s *LSym, size int64, flag int, pos src.XPos) {
262262
s.setFIPSType(ctxt)
263263
}
264264

265-
// EmitEntryLiveness generates PCDATA Progs after p to switch to the
266-
// liveness map active at the entry of function s. It returns the last
267-
// Prog generated.
268-
func (ctxt *Link) EmitEntryLiveness(s *LSym, p *Prog, newprog ProgAlloc) *Prog {
269-
pcdata := ctxt.EmitEntryStackMap(s, p, newprog)
270-
pcdata = ctxt.EmitEntryUnsafePoint(s, pcdata, newprog)
271-
return pcdata
272-
}
273-
274-
// Similar to EmitEntryLiveness, but just emit stack map.
265+
// EmitEntryStackMap generates PCDATA Progs after p to switch to the
266+
// liveness map active at the entry of function s.
275267
func (ctxt *Link) EmitEntryStackMap(s *LSym, p *Prog, newprog ProgAlloc) *Prog {
276268
pcdata := Appendp(p, newprog)
277269
pcdata.Pos = s.Func().Text.Pos
@@ -284,19 +276,6 @@ func (ctxt *Link) EmitEntryStackMap(s *LSym, p *Prog, newprog ProgAlloc) *Prog {
284276
return pcdata
285277
}
286278

287-
// Similar to EmitEntryLiveness, but just emit unsafe point map.
288-
func (ctxt *Link) EmitEntryUnsafePoint(s *LSym, p *Prog, newprog ProgAlloc) *Prog {
289-
pcdata := Appendp(p, newprog)
290-
pcdata.Pos = s.Func().Text.Pos
291-
pcdata.As = APCDATA
292-
pcdata.From.Type = TYPE_CONST
293-
pcdata.From.Offset = abi.PCDATA_UnsafePoint
294-
pcdata.To.Type = TYPE_CONST
295-
pcdata.To.Offset = -1
296-
297-
return pcdata
298-
}
299-
300279
// StartUnsafePoint generates PCDATA Progs after p to mark the
301280
// beginning of an unsafe point. The unsafe point starts immediately
302281
// after p.

‎src/cmd/internal/obj/ppc64/obj9.go

-8
Original file line numberDiff line numberDiff line change
@@ -1364,12 +1364,6 @@ func (c *ctxt9) stacksplit(p *obj.Prog, framesize int32) *obj.Prog {
13641364
p.To.Type = obj.TYPE_REG
13651365
p.To.Reg = REG_R22
13661366

1367-
// Mark the stack bound check and morestack call async nonpreemptible.
1368-
// If we get preempted here, when resumed the preemption request is
1369-
// cleared, but we'll still call morestack, which will double the stack
1370-
// unnecessarily. See issue #35470.
1371-
p = c.ctxt.StartUnsafePoint(p, c.newprog)
1372-
13731367
var q *obj.Prog
13741368
if framesize <= abi.StackSmall {
13751369
// small stack: SP < stackguard
@@ -1556,8 +1550,6 @@ func (c *ctxt9) stacksplit(p *obj.Prog, framesize int32) *obj.Prog {
15561550
p.To.Reg = REG_R2
15571551
}
15581552

1559-
// The instructions which unspill regs should be preemptible.
1560-
p = c.ctxt.EndUnsafePoint(p, c.newprog, -1)
15611553
unspill := c.cursym.Func().UnspillRegisterArgs(p, c.newprog)
15621554

15631555
// BR start

‎src/cmd/internal/obj/riscv/obj.go

-8
Original file line numberDiff line numberDiff line change
@@ -978,12 +978,6 @@ func stacksplit(ctxt *obj.Link, p *obj.Prog, cursym *obj.LSym, newprog obj.ProgA
978978
p.To.Type = obj.TYPE_REG
979979
p.To.Reg = REG_X6
980980

981-
// Mark the stack bound check and morestack call async nonpreemptible.
982-
// If we get preempted here, when resumed the preemption request is
983-
// cleared, but we'll still call morestack, which will double the stack
984-
// unnecessarily. See issue #35470.
985-
p = ctxt.StartUnsafePoint(p, newprog)
986-
987981
var to_done, to_more *obj.Prog
988982

989983
if framesize <= abi.StackSmall {
@@ -1070,8 +1064,6 @@ func stacksplit(ctxt *obj.Link, p *obj.Prog, cursym *obj.LSym, newprog obj.ProgA
10701064
}
10711065
jalToSym(ctxt, p, REG_X5)
10721066

1073-
// The instructions which unspill regs should be preemptible.
1074-
p = ctxt.EndUnsafePoint(p, newprog, -1)
10751067
p = cursym.Func().UnspillRegisterArgs(p, newprog)
10761068

10771069
// JMP start

‎src/cmd/internal/obj/s390x/objz.go

-10
Original file line numberDiff line numberDiff line change
@@ -327,7 +327,6 @@ func preprocess(ctxt *obj.Link, cursym *obj.LSym, newprog obj.ProgAlloc) {
327327
if !p.From.Sym.NoSplit() {
328328
p, pPreempt, pCheck = c.stacksplitPre(p, autosize) // emit pre part of split check
329329
pPre = p
330-
p = c.ctxt.EndUnsafePoint(p, c.newprog, -1)
331330
wasSplit = true //need post part of split
332331
}
333332

@@ -657,12 +656,6 @@ func (c *ctxtz) stacksplitPre(p *obj.Prog, framesize int32) (pPre, pPreempt, pCh
657656
p.To.Type = obj.TYPE_REG
658657
p.To.Reg = REG_R3
659658

660-
// Mark the stack bound check and morestack call async nonpreemptible.
661-
// If we get preempted here, when resumed the preemption request is
662-
// cleared, but we'll still call morestack, which will double the stack
663-
// unnecessarily. See issue #35470.
664-
p = c.ctxt.StartUnsafePoint(p, c.newprog)
665-
666659
if framesize <= abi.StackSmall {
667660
// small stack: SP < stackguard
668661
// CMPUBGE stackguard, SP, label-of-call-to-morestack
@@ -743,7 +736,6 @@ func (c *ctxtz) stacksplitPost(p *obj.Prog, pPre, pPreempt, pCheck *obj.Prog, fr
743736
spfix.Spadj = -framesize
744737

745738
pcdata := c.ctxt.EmitEntryStackMap(c.cursym, spfix, c.newprog)
746-
pcdata = c.ctxt.StartUnsafePoint(pcdata, c.newprog)
747739

748740
// MOVD LR, R5
749741
p = obj.Appendp(pcdata, c.newprog)
@@ -770,8 +762,6 @@ func (c *ctxtz) stacksplitPost(p *obj.Prog, pPre, pPreempt, pCheck *obj.Prog, fr
770762
p.To.Sym = c.ctxt.Lookup("runtime.morestack")
771763
}
772764

773-
p = c.ctxt.EndUnsafePoint(p, c.newprog, -1)
774-
775765
// BR pCheck
776766
p = obj.Appendp(p, c.newprog)
777767

‎src/cmd/internal/obj/x86/obj6.go

+4-17
Original file line numberDiff line numberDiff line change
@@ -1106,11 +1106,6 @@ func stacksplit(ctxt *obj.Link, cursym *obj.LSym, p *obj.Prog, newprog obj.ProgA
11061106
p.To.Offset = 3 * int64(ctxt.Arch.PtrSize) // G.stackguard1
11071107
}
11081108

1109-
// Mark the stack bound check and morestack call async nonpreemptible.
1110-
// If we get preempted here, when resumed the preemption request is
1111-
// cleared, but we'll still call morestack, which will double the stack
1112-
// unnecessarily. See issue #35470.
1113-
p = ctxt.StartUnsafePoint(p, newprog)
11141109
} else if framesize <= abi.StackBig {
11151110
// large stack: SP-framesize <= stackguard-StackSmall
11161111
// LEAQ -xxx(SP), tmp
@@ -1135,7 +1130,6 @@ func stacksplit(ctxt *obj.Link, cursym *obj.LSym, p *obj.Prog, newprog obj.ProgA
11351130
p.To.Offset = 3 * int64(ctxt.Arch.PtrSize) // G.stackguard1
11361131
}
11371132

1138-
p = ctxt.StartUnsafePoint(p, newprog) // see the comment above
11391133
} else {
11401134
// Such a large stack we need to protect against underflow.
11411135
// The runtime guarantees SP > objabi.StackBig, but
@@ -1158,8 +1152,6 @@ func stacksplit(ctxt *obj.Link, cursym *obj.LSym, p *obj.Prog, newprog obj.ProgA
11581152
p.To.Type = obj.TYPE_REG
11591153
p.To.Reg = tmp
11601154

1161-
p = ctxt.StartUnsafePoint(p, newprog) // see the comment above
1162-
11631155
p = obj.Appendp(p, newprog)
11641156
p.As = sub
11651157
p.From.Type = obj.TYPE_CONST
@@ -1189,8 +1181,6 @@ func stacksplit(ctxt *obj.Link, cursym *obj.LSym, p *obj.Prog, newprog obj.ProgA
11891181
jls.As = AJLS
11901182
jls.To.Type = obj.TYPE_BRANCH
11911183

1192-
end := ctxt.EndUnsafePoint(jls, newprog, -1)
1193-
11941184
var last *obj.Prog
11951185
for last = cursym.Func().Text; last.Link != nil; last = last.Link {
11961186
}
@@ -1202,9 +1192,8 @@ func stacksplit(ctxt *obj.Link, cursym *obj.LSym, p *obj.Prog, newprog obj.ProgA
12021192
spfix.As = obj.ANOP
12031193
spfix.Spadj = -framesize
12041194

1205-
pcdata := ctxt.EmitEntryStackMap(cursym, spfix, newprog)
1206-
spill := ctxt.StartUnsafePoint(pcdata, newprog)
1207-
pcdata = cursym.Func().SpillRegisterArgs(spill, newprog)
1195+
spill := ctxt.EmitEntryStackMap(cursym, spfix, newprog)
1196+
pcdata := cursym.Func().SpillRegisterArgs(spill, newprog)
12081197

12091198
call := obj.Appendp(pcdata, newprog)
12101199
call.Pos = cursym.Func().Text.Pos
@@ -1229,9 +1218,7 @@ func stacksplit(ctxt *obj.Link, cursym *obj.LSym, p *obj.Prog, newprog obj.ProgA
12291218
progedit(ctxt, callend.Link, newprog)
12301219
}
12311220

1232-
// The instructions which unspill regs should be preemptible.
1233-
pcdata = ctxt.EndUnsafePoint(callend, newprog, -1)
1234-
unspill := cursym.Func().UnspillRegisterArgs(pcdata, newprog)
1221+
unspill := cursym.Func().UnspillRegisterArgs(callend, newprog)
12351222

12361223
jmp := obj.Appendp(unspill, newprog)
12371224
jmp.As = obj.AJMP
@@ -1244,7 +1231,7 @@ func stacksplit(ctxt *obj.Link, cursym *obj.LSym, p *obj.Prog, newprog obj.ProgA
12441231
q1.To.SetTarget(spill)
12451232
}
12461233

1247-
return end, rg
1234+
return jls, rg
12481235
}
12491236

12501237
func isR15(r int16) bool {

‎src/runtime/preempt.go

+1
Original file line numberDiff line numberDiff line change
@@ -173,6 +173,7 @@ func suspendG(gp *g) suspendGState {
173173
// _Gscan bit and thus own the stack.
174174
gp.preemptStop = false
175175
gp.preempt = false
176+
gp.preemptRecent = true
176177
gp.stackguard0 = gp.stack.lo + stackGuard
177178

178179
// The goroutine was already at a safe-point

0 commit comments

Comments
 (0)