8/14/2019 PCI Express System Architecture_Mindshare Book
1/1103
N L-K -M
-
O K-N M
-
8/14/2019 PCI Express System Architecture_Mindshare Book
2/1103
- -
--
- - -- - - - ----
- - - - - - - - -
- - K -- - -- -- - - - - --
- - - - - --
K- - -- - - - - -- - --
--v - -v - v
v - v - v - v - -- v -
-- - - - - -
-
-- - -- - - - - - - - --
8/14/2019 PCI Express System Architecture_Mindshare Book
3/1103
---
-
-
- - {
{ - { { {
8/14/2019 PCI Express System Architecture_Mindshare Book
4/1103
-- - - -- --
- - - - -- - -- - -- - -
- -- - -- -- -- - --- - -- - - - - -
--
w
- - - - -- - - - -- - - - --
- -
-- - -- - -- - - -
- -- --
8/14/2019 PCI Express System Architecture_Mindshare Book
5/1103
To my parents Aruna and Shripal Budruk
who started me on the path to Knowledge
8/14/2019 PCI Express System Architecture_Mindshare Book
6/1103
8/14/2019 PCI Express System Architecture_Mindshare Book
7/1103
-
- - - - -
-- - -- -
- - -
- --
- -
-- --
- - - - - --
- - - - -
- - - - -
-
8/14/2019 PCI Express System Architecture_Mindshare Book
8/1103
-
-- - - - - --
- - - - - -
- - - -
- - - - -
-- - - - - -- -- -- -- - --- - --- - - - -
--
- -
-- -
-- -
- -- - - -- -
8/14/2019 PCI Express System Architecture_Mindshare Book
9/1103
-
-- --
-- - - -- - - -- - -- - -- - -- --
- -- -
- -- -
- - -
- -- - - - -
-- ---
- - -- ---
- - - --
-
- - --- - - - -
--
8/14/2019 PCI Express System Architecture_Mindshare Book
10/1103
-
-
- - - - - - -
- - - -
- -
- - -
-- -
-
-- - -
- - - -- - --- - -
- - - -
- -- - - -- -- - - - -
-- - - -
- -- -
-- - ---
--- -- - --- -
8/14/2019 PCI Express System Architecture_Mindshare Book
11/1103
-
-
- --
-- -- - - --
- -- - --
- -- - -- -
- --
- - -
- - - -
--- - --- - - -- - - -- - -
- -
- --
- -- -- - - - - -
- --
- -- - --
8/14/2019 PCI Express System Architecture_Mindshare Book
12/1103
-
- --
- -- - - - -
- -- -
- - - - - - -
- - - -- --- -- -- - - -
- -
- - -
- - - - -- - -
- - - -- -- -
-- - - - -
-- - - - - -
--
- - - - -
-
8/14/2019 PCI Express System Architecture_Mindshare Book
13/1103
-
- -
- - -- - - -- -
-- -- - -- - - -- - --- --- -- --
-- -
- - - -
- -
- -
- -
- -
- -
- - -- -
- -
8/14/2019 PCI Express System Architecture_Mindshare Book
14/1103
-
-
-
- -
-- --
- - --
-- -- - --
- -
--
- -
- - -
- --
8/14/2019 PCI Express System Architecture_Mindshare Book
15/1103
-
-
-
- - -
- -
- -
- - -
--
--
--
-- -
- -- -- -- -- -
- - --- -
-- - - -- - - -- -
8/14/2019 PCI Express System Architecture_Mindshare Book
16/1103
-
- -
--
-
-
- -
- -
- - -
-
-
-
- -- - - -- --
-
- - - - -- --- -
-
8/14/2019 PCI Express System Architecture_Mindshare Book
17/1103
-
-
- - -
- - -- -
- -
- - --- - --
- - - -
- - -
--
- - -- -
- -- - -- -- - -- -
-- -- -
- ---
-- - -
8/14/2019 PCI Express System Architecture_Mindshare Book
18/1103
-
- -- - --
- -
- - --
- -
- - - - -
--
-- --- - - - -
-- --
- -- - - - - - - -
- -
--- -
- -
-
8/14/2019 PCI Express System Architecture_Mindshare Book
19/1103
-
---
- - --
- -- -- -
- - - - -
- - -- --
-- -
-
-
- -- - - - - -
-
- -
- - -
- -
8/14/2019 PCI Express System Architecture_Mindshare Book
20/1103
-
- - -
- - -
- --- -
-
- - - -
- - -
- ---
- -
-
- - -- - -
- - -
8/14/2019 PCI Express System Architecture_Mindshare Book
21/1103
-
- -
-
-
- - -
- -
- - - - - -
- -
-
- -
- -
- --
- -
-
8/14/2019 PCI Express System Architecture_Mindshare Book
22/1103
-
- - - -
-
-
- -
- - -
-
- -- -- - -
--- -- -- --
- -- - -- --
-
- -
- - -
- - --
8/14/2019 PCI Express System Architecture_Mindshare Book
23/1103
-
- - -
-- - -
- - - - -
- - -
- - -
- - -- - - - - -
- - - - - -
- -
- -
-
- -
- -
8/14/2019 PCI Express System Architecture_Mindshare Book
24/1103
-
- - -
- -
-- - --
-
- - - - -
- - - -
8/14/2019 PCI Express System Architecture_Mindshare Book
25/1103
-
- - -
-- -
- -
- -
-
-
- - - - -
- -
-- - -
-
- - -
8/14/2019 PCI Express System Architecture_Mindshare Book
26/1103
-
-- - -- -- -
-
-- - -- - --- -- --
- - -
- - -- -
- - -
- -
-- - --
- - - - -
- - - -
- - -
8/14/2019 PCI Express System Architecture_Mindshare Book
27/1103
-
- -
- - - - - - - - - -
- - -
- --- - - - - - - - - - -
- -- - -
- - - - -
- - - -
-
- - - - - -
-
--
8/14/2019 PCI Express System Architecture_Mindshare Book
28/1103
-
-- --
-
-
--
- - --
- - -
- - -
- - -- -
- - -- - -
- -
-
-
8/14/2019 PCI Express System Architecture_Mindshare Book
29/1103
-
-
- - - -
- ---
--- -- -- - ---
-
- -
- - - -
-
- -
-
-
-
- - -- -
-
- --
8/14/2019 PCI Express System Architecture_Mindshare Book
30/1103
-
--
- - - --
--
- - -- -- --
-- -- --
-- - - -
--
- -
- -- - -
- - - -
- - -
-- - -
- - -
8/14/2019 PCI Express System Architecture_Mindshare Book
31/1103
-
--
-- ---
- --
-- --
--
--
- - -
-- - --
- --
- --
-- -- - --
- - - - -- -
-- - - -
-- -- - --
8/14/2019 PCI Express System Architecture_Mindshare Book
32/1103
-
--
-- -- -- - -
- - - - -- -
-- - -- -- -- - - - - --
- - - -
- - - - - -
- - - - -- -
- - - - - - -
- - --
- - - --
- -- -- -
- -- -
8/14/2019 PCI Express System Architecture_Mindshare Book
33/1103
-
- --
- -- - - -
-- -- -
-- - -- - -- -
- - - --
-- -- - --
- - - - - - - -- - - -
-- - - - - -- - - - - - - - - - -
- -- --
- - -- - -
8/14/2019 PCI Express System Architecture_Mindshare Book
34/1103
-
-
- - -
- -- -
-- - - -
-- - -- --
- - - - - ---
- --
- -
- -- - - - - -
- --
- - -
-- - -
-
- - -
- -
8/14/2019 PCI Express System Architecture_Mindshare Book
35/1103
-
-
- - - - -- - -
-- - - - - - - - -
--- -- --- --
-- --
-- - - --- --- --
--- - -- --
- -
- - - - --
-
- - -
-- -
8/14/2019 PCI Express System Architecture_Mindshare Book
36/1103
-
- -- - -
- - - - - - - - - - --- - - - -
-- --
-- - --
-- - - -- - -
- - - - - --
- -
- - -
-- - - - - -
-- - - -
-- -
-
8/14/2019 PCI Express System Architecture_Mindshare Book
37/1103
-
- - - - - - - - - - - - - - - - - --
- - - -
- - - -
-
- - -- - --- - --
-
- - - -- - - -- -- -- -
-- -
8/14/2019 PCI Express System Architecture_Mindshare Book
38/1103
8/14/2019 PCI Express System Architecture_Mindshare Book
39/1103
-
- - --
- - - - - - - - - - - - - -- - - -- - -
- - - - - - - -- -- -- - -- - - -- - -- - - - - - - - - -
- -- - - - - - -- - - -- --- - -- --- -
-- - - - --
8/14/2019 PCI Express System Architecture_Mindshare Book
40/1103
-
--
- - - - - - - - - - - - - - -- - --- -- - -- - -- - - -
- - - -- - -- - -- - - - -- - - -- - - - - -- - -- - - -- -
- - - - - - - -- - -- ----- - - - - - - - - - - - - -
-- - -
8/14/2019 PCI Express System Architecture_Mindshare Book
41/1103
-
- - - -- - -- - - - - - -
- - - - - - - - - - -- -- - - - -
- - - - - - - - - - - -
8/14/2019 PCI Express System Architecture_Mindshare Book
42/1103
-
-
- - - - - - - - - - -- - - -- -- -- -
- -- - -- - - - - - - - - - - --- -- -- -- -- -- - -- - - -- -- --
- - - --- - - - - - -- - - - - - - - - - - - - - -- - - -
- - - - -
8/14/2019 PCI Express System Architecture_Mindshare Book
43/1103
-
- -
- - - - - - - - - - - - - -
- --- - - - - -
- - - - - --- -- - -- - - - - --
- --
--
8/14/2019 PCI Express System Architecture_Mindshare Book
44/1103
-
- - - - - - - - - - - - - - - - - - --
- - - - - - -
- - - - - - - - -- - - - -- -- -
-- - - -- - --
- - - - -- --
8/14/2019 PCI Express System Architecture_Mindshare Book
45/1103
-
- --
- - - - -- -- - -- - - - - - -- - - - -
- - - - - -- -- -
-- - -- - - - - -- - - -- -- - - - -
- - - - - - - - -- -- -- - - - - -- - -- - --
-- -
8/14/2019 PCI Express System Architecture_Mindshare Book
46/1103
-
--
- - -- - - - - - - -- - - - - -
- - - - - -- - - --
- -- - - - - - - -- - -- - - - - -- - -- - -- - -- - -- - -- - - - - - -
- - - -
8/14/2019 PCI Express System Architecture_Mindshare Book
47/1103
-
-
- - - - - - -- --- -- --- - - - -- - - --- --- - - -- - -- -
- - - -- - -- - - - - - - - - - - - - - - -
- - - - - - - - - - - - - - - - - - - - -
-- -
8/14/2019 PCI Express System Architecture_Mindshare Book
48/1103
-
- - - - - - - - - - - - - - - -
- - - -- -- - - -- - - -
- - - - - -- - -- - - - - - - -- - - - -- - - - -- -- - - -- - -- - -- - -- - --
- - - -- - - - -
8/14/2019 PCI Express System Architecture_Mindshare Book
49/1103
-
- -- --
- -- -- - -- - -- - - - - - -- - - - - --
8/14/2019 PCI Express System Architecture_Mindshare Book
50/1103
8/14/2019 PCI Express System Architecture_Mindshare Book
51/1103
-
-
- - - - - - - -- - - -- - - -- -- - -- - - - - -- -- - - -- - - -- -- - - - -- - -
-- - -- - - - - - - - - - - -- -- - - - - - - - - - - - - - - -- - - -- -- -- -- -- -- - - -
- - -
8/14/2019 PCI Express System Architecture_Mindshare Book
52/1103
-
-
-- -- - - - - - - -- - -- - -- - - - - - - - - - - -- - - -
- - - - -- -- - - - - - - - - - - - - - - -- - - - - - - - -- - - - --- - - --- - - -
-- -
8/14/2019 PCI Express System Architecture_Mindshare Book
53/1103
-
-
- - -- - - -- - - - -- - -- -- - - -- - -- --
-- - - -- -- - - - - - -- - -- - - - - - - - - - - - - - - - - - - - - - -- - --
- -- - - - -
8/14/2019 PCI Express System Architecture_Mindshare Book
54/1103
-
-
- - - - - - - - - - - - - - - - - - - - - -
- - - - - - - - - -- - -- -- -- - -- - -- - - -- - -- - -- - -- - - -- - - - -- - -- - -- --- -- - - -- -- - -- - -- - - -- - -- - -- - -
8/14/2019 PCI Express System Architecture_Mindshare Book
55/1103
-
- - - - -
- - -
- - - --
- -
-
- - -- - -
- - - - -- - - --
--
- - -- - -
- -
8/14/2019 PCI Express System Architecture_Mindshare Book
56/1103
8/14/2019 PCI Express System Architecture_Mindshare Book
57/1103
- -- - - - -- - - --
-
--
-
-- -
-
-
-
-
-
- -
-
-
-
- - -
- -
-
-- -
-
-
8/14/2019 PCI Express System Architecture_Mindshare Book
58/1103
-- -
- - -- - - - - - -- - - - - -- - -
- -- - - - -- - - -- - - -- - -
- - -- -- -- - - - - - - -
- - - - - -- - - - - - -
-
- -
- -
-
-
-
-
- -
-
8/14/2019 PCI Express System Architecture_Mindshare Book
59/1103
-
-
- - - - - -- - - - - - - - --- -
-
- - - -
- - - -- - -- - - - -- -- - - - - - - - -- - - - - - ---- - ---- - -- - - -- ---- -- -- - - - --
- - - -- - - --
-- -- -- -
8/14/2019 PCI Express System Architecture_Mindshare Book
60/1103
-- -
-
- - - - -
--
-- - - - -- - - - - --
- -
- -
- - - -
8/14/2019 PCI Express System Architecture_Mindshare Book
61/1103
-
- -- -
- -- - - -
-- --- -
-
- - - - - - - -
-
-
- - -- - - -
-
- -- -- - -
-- - - --- - ---
- -- - - - -
- - --
-
8/14/2019 PCI Express System Architecture_Mindshare Book
62/1103
-- -
- - --- -
-- - -- - - -
--
- -
8/14/2019 PCI Express System Architecture_Mindshare Book
63/1103
8/14/2019 PCI Express System Architecture_Mindshare Book
64/1103
8/14/2019 PCI Express System Architecture_Mindshare Book
65/1103
--
- -- - - -- - - - -- -- - -- -- - - -- --- -- --- -- - - - -- -- - -- -- - -- -- -- -- -
-- - -- - -- - - -
- --- --
--
-- - - - - - - - - - -- -- - -- -- - -- - - - - - - -
8/14/2019 PCI Express System Architecture_Mindshare Book
66/1103
-- -
-
-- - - - - - - - - -
-- - - - - - -- -- --- - -- -- - - - - -- -- -- - -
- - -- -- - - -- - - - --- -- -- - - - - - -
-- -- -- -- - - - - - - -- - - - - -
-
- -- - - - -- - - - -- -- -- -- - - -
-- - - - -- - - -- - - -- - --- - - - ----- - -- -
- -- - - -- - - - - -- - - - - -
8/14/2019 PCI Express System Architecture_Mindshare Book
67/1103
-
- - - - - -
-
-- - -- - - - -- - - --- - - - - --- - - - - -- - - - - - - - - -- - -- - - - -- -
-- - - -- -- - - -- - - -
-- - -- -- - - - - - -- - -
-
- -
-- --
- - -- -- - - -- - - - - - - -- - - - -- -- --
8/14/2019 PCI Express System Architecture_Mindshare Book
68/1103
-- -
- -
- -- - - -- - - - - - - - -- - - - - - - - -- -- --
- -- - - - - - - - - - - -
- - -
-- - - - -- - - - - - - - - -- - - - -- -
- - - - - - - -
- - - -
- - -
--
8/14/2019 PCI Express System Architecture_Mindshare Book
69/1103
-
--
-- - - - - --- - - - -- - - - - - -
-- - - - - - - -- - - - -
- - - - --- - -- - --- --- - - - - - - - - - - - - -- ---
- - -
-
- -
--
--
--
--
--
-
- - - -
8/14/2019 PCI Express System Architecture_Mindshare Book
70/1103
-- -
-- - - - - -
-- - --- -- -
- -- - -
- - -- - - - -- - - -- -
- - -- -- -- - - - - - - - - - -- -- --- - - - - - -
-- - -
--
--
8/14/2019 PCI Express System Architecture_Mindshare Book
71/1103
-
- - --
1.6 3.23.2
6.4
12.8
25.6
100
0
10
20
30
40
50
60
70
80
90
100
8/14/2019 PCI Express System Architecture_Mindshare Book
72/1103
-- -
- -
- - -
- - - -- -- --- - - - - - -- - -- - -
- - - - - - - -
- - - -
- - - - - - - - - -- - - - - - -- - - - -- - - -
- - - - - - -- -- - - - - - - - - -
-- - - -
-- - - - -- - -- -- - - - - - - - -- - -- - - - -- -
- - - -- - - - - --- - - --- - - - --- -- -- -- - - --
- -- - - - -
- -- - - - - - -
8/14/2019 PCI Express System Architecture_Mindshare Book
73/1103
-
-
- -- - - - - - -
- - - - - - - - -
- - - - - - - - - - -
- --- - --- - -- - - - - - - - - -
- -
Processor
North Bridge
(Intel 440) SDRAM
South Bridge
AGP2x
PCI-33MHz
IDEHDD
USB
GFX
CD
Super
IO
COM1
COM2
Super
IO
COM1
COM2
SCSI
Audio
Chip
Modem
Chip
ISA
Boot
ROM
FSB
Ethernet
Slots
Arbiter
8/14/2019 PCI Express System Architecture_Mindshare Book
74/1103
-- -
- - -
- - - - - - - - - - -- -
- - - - - - - - - - - -- - - - - - - - - - - - -
- -
1 2 3 4 5 6 7 8
CLK
FRAME#
AD
C/BE#
IRDY#
TRDY#
DEVSEL#
Addr Data1Data
2Data
3
BusCmd
ByteEnables
ByteEnables
ByteEnables
GNT#
AddressPhase
Data Phase 1 Data Phase 2 Data Phase 3
WaitState
WaitState
WaitState
8/14/2019 PCI Express System Architecture_Mindshare Book
75/1103
-
- -- -
- - - - - - - - -- -- -- - - - - --
-
- - - - - - - - - - - - - - - - -- - -- - - -
- - - - - - - - - - - -- - -
- -
Processor
North Bridge
(Intel 440) SDRAM
South Bridge
AGP2x
PCI-33MHz
IDEHDD
USB
GFX
CD
Super
IO
COM1
COM2
SCSI
Audio
Chip
Modem
Chip
ISA
Boot
ROM
FSB
Ethernet
Slots
PCIBridge
PCI Bus 0
PCI Bus 1
Ethernet
Processor
North Bridge
(Intel 440) SDRAM
South Bridge
AGP2x
PCI-33MHz
IDEHDD
USB
GFX
CD
Super
IO
COM1
COM2
Super
IO
COM1
COM2
SCSI
Audio
Chip
Modem
Chip
ISA
Boot
ROM
FSB
Ethernet
Slots
PCIBridge
PCI Bus 0
PCI Bus 1
Ethernet
8/14/2019 PCI Express System Architecture_Mindshare Book
76/1103
-- -
-- - -
-- - - - - - - - - - -- - -- - - - - - - - - - - - - - - -- - - - - - - -
-
- - - - - -- - -
--
-
Processor
North Bridge(Intel 440) SDRAM
South Bridge
AGP
2x
PCI-33MHz
IDEHDD
USB
GFX
CD
Super
IO
COM1
COM2
Super
IO
COM1
COM2
SCSI
Audio
Chip
Modem
Chip
ISA
Boot
ROM
FSB
Ethernet
Slots
1) Programmed IO
2) DMA
3) Peer-to-Peer
8/14/2019 PCI Express System Architecture_Mindshare Book
77/1103
-
- - -
- - - - - - - - - - - - - -- - - -- -- - - - --- - - - -
- --
- -- - - -
- -- -- -- - --- - - -
- - - -- - -- - - - - - - - - - - - -- - -- - - - - - - - -
- - - -- - - - - - -
- - - - - - - - - - - - - - -- - - - - - - - - - - -- - --- - - - - - - - - - - - -
8/14/2019 PCI Express System Architecture_Mindshare Book
78/1103
-- -
-
- - - - - - - - -- - - - - - -- - - -- - - - - - - - -
-
-- - - - - - - - - - - -
- - -- - - - - - - - - -
- - - - - - - - -- - - -- - - -
-
Processor
North Bridge
(Intel 440) SDRAM
South Bridge
AGP
2x
PCI-33MHz
IDEHDD
USB
GFX
CD
SCSI
ISA
FSB
Ethernet
Slots
Arbiter
8/14/2019 PCI Express System Architecture_Mindshare Book
79/1103
-
-
- - - -- - -- - - - - -
- - - - - - - - - - - - - -- - -- - - -
- - - - - - - - - -
- -
Processor
North Bridge(Intel 440) SDRAM
South Bridge
AGP2x
PCI-33MHz
IDEHDD
USB
GFX
CD
SuperIO
COM1
COM2
SuperIO
COM1
COM2
SCSI
AudioChip
ModemChip
ISA
BootROM
FSB
Ethernet
Slots
Arbiter
1. Initiate
3. Retry
2. Target devicenot ready
8/14/2019 PCI Express System Architecture_Mindshare Book
80/1103
-- -
- - - - -
-- -- - - - - - - - - - - - - -
- - -- - - - - - - - - -
- - - - - - - -- - -- - - --- --
- - - - -- - - -
- -- - - - -- -
- - -
Processor
North Bridge
(Intel 440) SDRAM
South Bridge
AGP2x
PCI-33MHz
IDEHDD
USB
GFX
CD
Super
IO
COM1
COM2
Super
IO
COM1
COM2
SCSI
Audio
Chip
Modem
Chip
ISA
Boot
ROM
FSB
Ethernet
Slots
Arbiter
1. Initiate3. Disconnect
2. Some data
transferred
8/14/2019 PCI Express System Architecture_Mindshare Book
81/1103
-
- - -
- - - -- - - - - - - - - - -- - - - -- - - - - - - - - - - - - --- - - - - -- - - - - - - - - -- - - - - - - - - --
- - - - - - -- - - - - - - - - - - - - - - -- -- - -- - - --- -- -
- - - - -- - --- -
- -- -- - - -- - - - -- - - -- - -- - - - - - - - - - - - - - - - - - - - -
8/14/2019 PCI Express System Architecture_Mindshare Book
82/1103
-- -
- - -- - - -- - - - - -- -- - -- - -- - - - - - -- - --- - - --- -- - - --- --- - - -
- -- -- -- -
Processor
North Bridge
(Intel 440) SDRAM
South Bridge
AGP
2x
PCI-33MHz
IDEHDD
USB
GFX
CD
Super
IO
COM1
COM2
Super
IO
COM1
COM2
SCSI
Audio
Chip
Modem
Chip
ISA
Boot
ROM
FSB
Ethernet
Slots
Arbiter
Interrupt
Controller
INTR
INTx#
INTx#
8/14/2019 PCI Express System Architecture_Mindshare Book
83/1103
-
- - - - - - -- - - - - -- - - -- -- - -
--
-- -- -- - - -- -- -- - - --- - -- - --- - -- -- -- - - - - - -- - - -- -
Processor
North Bridge(Intel 440) SDRAM
South Bridge
AGP
2x
PCI-33MHz
IDEHDD
USB
GFX
CD
Super
IO
COM1
COM2
Super
IO
COM1
COM2
SCSI
Audio
Chip
Modem
Chip
ISA
Boot
ROM
FSB
Ethernet
Slots
Arbiter
Error
Logic
NMI
PERR#
SERR#
8/14/2019 PCI Express System Architecture_Mindshare Book
84/1103
-- -
-- - -- - - -- - -- - - - - -
- - - - -- -- -- -- - - - -- - - - -- - -- - -- - -- -
--
Boot ROM
Expansion ROM
Legacy Video
Extended
Memory
Conventional
Memory
1MB
AGP Video
PCI
Memory
PCI
Memory
DRAM Boundary
4GB / 16 EB
Memory Map
IO Map
Legacy IO1KB
Data Port
Address Port CF8h-CFBh
CFCh-CFFh
64KB
PCI IO
Space
PCI
Configuration
Space16MB
256B
256B
256B
640KB
8/14/2019 PCI Express System Architecture_Mindshare Book
85/1103
-
- --
- -- -- -- - - -- --
- - - -
-- - -- - - - -- - - -- - - -- - - - - -- - - -
Processor
North Bridge
(Intel 440) SDRAM
South Bridge
AGP
2x
PCI-33MHz
IDEHDD
USB
GFX
CD
SuperIO
COM1
COM2
SuperIO
COM1
COM2
SCSI
Audio
Chip
Modem
Chip
ISA
Boot
ROM
FSB
Ethernet
Slots
Address Port Data Port
IDSEL
Config.
Reg.
Config.Reg.
Config.Reg.
8/14/2019 PCI Express System Architecture_Mindshare Book
86/1103
-- -
-
- - - - - - -- - - -- -- - -- - -
- - -- -- - -- - - - -- - --- - --- --- -- - - -- - -- - - --
-
Reserved
Reserved
HeaderType
Expansion ROMBase Address
Base Address 5
Base Address 0
Base Address 1
Base Address 2
Base Address 3
Base Address 4
VendorID
DeviceID
LatencyTimer
00
01
02
03
04
05
06
07
08
09
10
11
12
13
14
15
RevisionID
Class Code
InterruptLine
InterruptPin
Min_GntMax_Lat
BISTCacheLineSize
StatusRegister
CommandRegister
Required configuration registers
DoublewordNumber
(in decimal)
0123
Byte
Subsystem ID SubsystemVendor ID
CardBus CIS Pointer
CapabilitiesPointer
192 Byte
Device Specific
Configuration Register Space
64 Byte
PCI Configuration Header
(Two formats: Header 0
and Header 1)
256 Byte Configuration
Register Space Per Function Header 0
8/14/2019 PCI Express System Architecture_Mindshare Book
87/1103
-
-- - - - - -- - -- -- - -- - - - -- - - - - - - -
- -- - - - - - -- - - -- - -- --
- - - - -- -
- - - -- - -- --
-- - - - - -
-- --- - - - --- -- -- - -- - -
-- --
- -
- - - -- - --
- - - - -- - - -
8/14/2019 PCI Express System Architecture_Mindshare Book
88/1103
8/14/2019 PCI Express System Architecture_Mindshare Book
89/1103
-
- - -
--- -- - - -- - --
- - -
-- - - -- - --- - - - - -- -- - -- -- - -- - -
- -
Processor
Memory Controller Hub
(Intel 860 MCH)
RDRAM
IO Controller Hub
(ICH2) IEEE
1394
AGP
4x
PCI-33MHz
Hub LinkIDE
HDD
USB 2.0
LPC
GFX
CD
Super
IO
COM1
COM2
Super
IO
COM1
COM2
SCSI
AudioCodec
ModemCodec
AC97
Link
BootROM
Processor
FSB
Ethernet
RDRAM
P64H
Hub Link
P64H
PCI-66MHz
Slots
Slot
8/14/2019 PCI Express System Architecture_Mindshare Book
90/1103
-- -
- -
- - -- - - - - - - - - - - - -- --- - -
- - - - - -- - - - - - - - - - - - - - - -- - - - - - - -- - -
- - - -- - - - - - -- - -- - - --
- - -- - - - -- - -- - - -
-
- -- - - -- - - - -
- - - - - -
- - - - - -- -- - - -- - - -
- - - - - - - -
-- - - --- - - - - - - - - - -- - - - -- - -
8/14/2019 PCI Express System Architecture_Mindshare Book
91/1103
-
- - --- -- - - --
- -- -- - ---- -- - -- -- -- - - - -
- - - -- -- -
-- - - - - - - - - - - - - - - - - - - - --
- - -
- - - - - - -- - - -
--- - -- -- - -- -- - - - - -- -- - - - - - - - -
- - - - - --
- - -
- - - - -- - - - -- - - - - -- - - - - - - - -- -- - - -- - - --
8/14/2019 PCI Express System Architecture_Mindshare Book
92/1103
-- -
-
- - - -
- -- - -- - - - - - - - - -- - --
-- - - - - - - - - - - -
- - - - -- - - - - - - -
- -
Processor
Memory Controller Hub(Intel 7500 MCH)
DDR SDRAM
IO Controller Hub(ICH3)
IEEE1394
PCI-33MHz
Hub Link 1IDE
USB
LPC
SCSIAC97Link
Boot
ROM
Processor
FSB
Ethernet
DDR SDRAM
P64H2Hub Link 2
P64H2
PCI-X
P64H264-bit,
66MHz or 100MHz or 133MHz
Hub Link 2
Slots
8/14/2019 PCI Express System Architecture_Mindshare Book
93/1103
-
- - -
- - - - -
- --
- - - - -- -- --
- - - - - - - - -- - - -- - - - - - - -- -- - - -
- - - - - - -- - - - - -
- - - - - - - - - - -- -
- -
1 2 3 4 5 6 7 8 9
CLK
FRAME#
AD[31:0]
C/BE#[3:0]
IRDY#
TRDY#
DEVSEL#
10 11 12
Address ATTR
Cmd ATTR
Data-0 Data-1 Data-2 Data-3
AddressPhase
AttributePhase
ResponsePhase
DataPhase
1
DataPhase
2
DataPhase
3
DataPhase
4
TurnaroundCycle
DecodeA
BusIdle
8/14/2019 PCI Express System Architecture_Mindshare Book
94/1103
-- -
- -- - --
- - - - - - - - -- - - - - - - - - - - - - - - - - - - -- - - -
- - - - - -- - - - -- - - - - - -- - - - -
-- - - -
- - - - - - - -
- - - -- - - - -
-
Requester Completer
1. Requester initiatesread transaction
2. Completer unable to
return data immediately
3. Completer memorizes
transaction
4. Completer issues
split response
5. Completer initiates split completion
bus cycle to return read data
8/14/2019 PCI Express System Architecture_Mindshare Book
95/1103
-
- - - -
- -- - - - - - - - - - - - -
-- - - - --
-- - -- - - --
-- -- --
- -- -- - -- - -- - - -- - - - -- -- ---
- -- - - - -- - - - --
- - - - - - - - - -
- - -
-- -- - - -- -- - -- - - - - - - - - - -- - - -- - - --
-
8/14/2019 PCI Express System Architecture_Mindshare Book
96/1103
-- -
- - - -
- -- - - - - - -
- - - - - - - - - - --- - - - - - - -
- - - - -- - - - - - -- - - - -- --
- -- -- - - - -- - -
- -
Processor
Host Bridge
DDR/QDR
SDRAM
Processor
FSB
PCI-X
DDR/QDR
PCI-X 2.0
Bridge
PCI-X 2.0
Bridge
10Gb
Ethernet
10Gb
Ethernet
10Gb
Ethernet
10Gb
Ethernet
PCI-X
DDR/QDR
PCI-X
DDR/QDRPCI-X
DDR/QDR
SCSIRAID
Disk arrayPCI-X 1.0
N.B. PCI-X DDR/QDR buses are point-to-point parallel buses
Infiniband
PCI-X
DDR/QDR
PCI-X 1.0
Bridge
PCI-33MHz
S
IO
COM1
COM2
S
IO
COM1
COM2
Graphics
Slot
Slots
Slots
Slot
8/14/2019 PCI Express System Architecture_Mindshare Book
97/1103
-
--
-- - - - - - - - - -- - --
- - -- --- -- - - - --- - - --- - -- --- - - -
-- -- - - - - - - -
-- - - - -- -- -- - - - - - - - - - - -
--
PCI-XP
DeviceA
PCI-XP
DeviceB
Link (x1, x2, x4, x8, x12 x16 or x32)
Packet
Packet
8/14/2019 PCI Express System Architecture_Mindshare Book
98/1103
-- -
-- - -
- - - - - - - - - - - - - -
-
- - -
- --- - - - -- - - - - -
- - - -- --- - - - - --
-
- - - -- - -- - - - - - - -- - - - - - - - - - - - -- - -
- - - -- -- -- -- -- -
--
D+
D-
V DiffpVcm
8/14/2019 PCI Express System Architecture_Mindshare Book
99/1103
-
- - -- - - - -- - -- -- -
- -- - --- - - - -- -- -- -- -
--
-- -- - -- -- -
-- -- -- - - - - - - - -- - - -- - -- - --- - --- -- - --- - -- --- ---
-- --
-- -- - - - - - -- --
- -- -- - -- - - -- -
-- -
-- -- - - -- - - - - -- - - - - - - - - - -- - - - - - -- - - - -- - -
-
8/14/2019 PCI Express System Architecture_Mindshare Book
100/1103
-- -
--- -
- - --- - - - - - - - - - - -
- -
-- - -
- - - - - -- -- - -- -
--- - - -
-- - - - - - - - - -- - - - -
- - -
-- - -- --- - - - - - - - - -- - -
-- - - - - - - - - - - - -
- - - - -- - - -- - -- - --
8/14/2019 PCI Express System Architecture_Mindshare Book
101/1103
-
-- --
- - -
- - - - - --
- - - - - - - - - - - - - - - -- - -
- - - - - - - - - -- - - - -- - -
- - -- - - - - --- -
-- - -- - -- - -- -- - - - - -- - -
-- -- -- - --- -- - - -- - - - -- - - - - - - - - - -
- - - - - - - -
8/14/2019 PCI Express System Architecture_Mindshare Book
102/1103
-- -
- - -- -
- - - - -
-- -- -- - - - - -
-- - -- -- - - - - -
-- -- -- - - -- ---
- - -- - - - -- - - -- - - - -
-- - - - ---- - -- - - - - - - - -- - - - -- - -
- - - -- -- -- - -- - - - -- - -- -
-- -- --
- - - - - ---
-- - - -- - - - -
8/14/2019 PCI Express System Architecture_Mindshare Book
103/1103
-
-
-- -- - - - - -- - - - - - - --
- -- - -- -- - - - - - -- - -- -
-- -- - -- -- - - -- - - - -- - --- -- -- -- -- - -- -- - -- - - -
- -
- -- - - - - - -
- - - - - - - - -- -- -- - -- - - - - - - -- - - -- - -
- - - - - - -- - -
-- --- - -
- - - -- - -
8/14/2019 PCI Express System Architecture_Mindshare Book
104/1103
-- -
--
- -- -- - -- -
- - --- -- - -- - - -- - - - - - - - -- - - -- -
-- - - - - -- - - - -- -- - - - -
- - - - - - - - -
--
Switch
PCI-XPEndpoint
LegacyEndpoint
PCI-XPEndpoint
Root Complex
CPU
PCI-XP 1
Memory
PCI-XPBridge To
PCI-XP 6 PCI-XP 7
PCI-XP 4 PCI-XP 5
Legend
PCI Express Device Downstream PortPCI Express Device Upstream Port
PCI-XPEndpoint
Switch
Virtual
PCIBridge
Virtual
PCIBridge
Virtual
PCIBridge
VirtualPCI
Bridge
PCI/PCI-X
PCI/PCI-X
Bus 2
PCI-XP
3
Bus 8
Bus 0 (Internal)
8/14/2019 PCI Express System Architecture_Mindshare Book
105/1103
-
- -- - -
- - - - - - -
- - - -- - - -- - -- -- -
- - - -- -
- - -- -- - -- -- - - - - - - -- - - - -- - - - - -- - - - - -- - - -- - - - - - - - -- -- - - - -- - - -- -- - - - - -- - - - -- - - - --
-- - -- - -
- - - - -- - - - - --- - - - -
- - -- - - - - -- --
-- - --
8/14/2019 PCI Express System Architecture_Mindshare Book
106/1103
-- -
- - - - --
- - -
- -- - - - - - -
- -- --- -- - - - - - - - - - - - - -- - - -- - --
- -- - -- - - -- - -- -- --- - - - - - - - - - -
- - - - - - -- - - - - -- -- -- - -- - -- --
- - - - -- -- - - - - - - - -- - -- - -
- -- - -- - -- - - - --
-
-
-- -- - - - - - - - -- -
8/14/2019 PCI Express System Architecture_Mindshare Book
107/1103
-
- - - -- - -
- -- - -- - -- -- - - - - - - - -- - - - - - -- - -- - - --
- - - - - -- -- - - - -- -- --
-- -
- -- -
- - -- - -- - - -- - -- -- -- - -- - - - - - -- - - - - - - -- - - -- - - -- - -- -- -- - - - - - - - -
-- - -- - - -- - -- - -- - -- - - -- - - - - -- -- - - - - - -- -- - -- - - - -
- --
8/14/2019 PCI Express System Architecture_Mindshare Book
108/1103
-- -
- - - - -- --
- - - --
- -- -
- - -- -- - - - - - - -- -- - -- - -- - - - - - -- - -- - - - --
- -- -
Processor
Root Complex DDR
SDRAM
IO Controller Hub
(ICH) IEEE
1394
PCI Express
GFX
PCI
PCI ExpressSerial ATA
HDD
USB 2.0
LPC
GB
EthernetAdd-In Add-In Add-In
PCI Express
GFX
FSB
PCI ExpressLink
S
IO
COM1
COM2
S
IO
COM1
COM2
Slot
Slots
8/14/2019 PCI Express System Architecture_Mindshare Book
109/1103
-
-
-- -- - -- -- - - - - -- --
- - -- - - - - - -
- -- -
Root Complex
Existing
IO Controller Hub
(ICH4)
PCI-33MHz
Hub LinkIDE
HDD
USB 2.0
LPC
GFX
CD
Audio
Codec
Modem
Codec
AC97Link
Boot
ROMEthernet
Slots
DDRSDRAM
Processor
FSB
PCI Express
Links
S
IO
COM1
COM2
S
IO
COM1
COM2
PCI Express
IEEE
1394
8/14/2019 PCI Express System Architecture_Mindshare Book
110/1103
-- -
-- -
- - ---
-- - - -- - -- - - -- - -- -
- -- - -- - - -
- -- - -
-- -
Processor Processor
Root ComplexGFXDDR
SDRAM
10GbEthernet
PCI-XP-to-PCI
PCI
InfiniBand
SwitchOut-of-Box
SCSI
RAID Disk array
IEEE1394
InfiniBand
GbEthernet
PCI Express
Link
Switch Switch
Switch
PCI ExpressGFX
FSB
S
IO
COM1
COM2
S
IO
COM1
COM2
Endpoint
Endpoint
Endpoint Endpoint
Endpoint
Endpoint
10GbEthernet
Endpoint
Add-In
Add-In
Fiber
Channel
Slots
8/14/2019 PCI Express System Architecture_Mindshare Book
111/1103
-
- - - - -- - - - -- - -- -- - -
-- --- -- - -- -- - - - -- -- - -- -- - - - - -- -- -
-
- - -- - -- -- - - - - -- - - - - -- - - - - --
-- - -- -- -- - - -- --
-- --
-- - - - -- - - --- - - - -
8/14/2019 PCI Express System Architecture_Mindshare Book
112/1103
-- -
-- -- -
-- -- -- - -
-- - - -- -- - -- ---- - - - - -- - -- - -- - --
- -- - -- - - - - -- - - -- - - - - - - - - - - -- - -- -
- -- - -- - - - - -- -
- - - --- - - -- -
-- - - --
- - -
- -
-
-
-
-
-
-- -
8/14/2019 PCI Express System Architecture_Mindshare Book
113/1103
-- -
-- - - - - - -- - - - - - - - -- -- - -- - -
-- -
-
-
- --
-
-- - -
-- - -
-- --
-- --
8/14/2019 PCI Express System Architecture_Mindshare Book
114/1103
-- -
- --
-- - - - - - - - - -- - - - - - - - - - - -- - - --
- - -- - - - -- - -
- - - - - - - -- - - - -- - -
- -- - - - - -- - -
-- -- -- - - - - - - - - - --
- - - - - -- - - -
8/14/2019 PCI Express System Architecture_Mindshare Book
115/1103
- - --
-- - - - - - - - - -- - - - - - --
- - - -- - - - - - -- -- --
- -
Requester Completer
MRd, IORd,CfgRd0, CfgRd1
CplD or Cpl
Legend:MRd = Memory Read RequestIORd = IO Read RequestCfgRd0 = Type 0 Configuration Read Request
CfgRd1 = Type 1 Configuration Read RequestCplD = Completion with data for normal completion of MRd, IORd, CfgRd0, CfgRd1
Cpl = Completion without data for error completion of MRd, IORd, CfgRd0, CfgRd1
8/14/2019 PCI Express System Architecture_Mindshare Book
116/1103
-- -
- - - - - - - -- - -- - - - -- - -
- -- - -- -- -- - - - - - - - - - --
- - - - - -- - - - -- - - -
- - - -- -- ---- - - -- ---
- -
Requester Completer
MRdLk
CplDLk or CplLk
Legend:
MRdLk = Memory Read Lock RequestCplDLk = Locked normal Completion with data for normal completion of MRdLk
CplLk = Locked error Completion without data for error completion of MRdLk
8/14/2019 PCI Express System Architecture_Mindshare Book
117/1103
- --
-- - - - - - - - - -- - - - - - - - - - - -- -- - -- -- --
- - -- - - -
- - - - -
- - - - - -
- -
Requester Completer
IOWr, CfgWr0, CfgWr1
Cpl
Legend:
IOWr = IO Write RequestCfgWr0 = Type 0 Configuration Write RequestCfgWr1 = Type 1 Configuration Write Request
Cpl = Completion without data for normal or error completion of IOWr, CfgWr0, CfgWr1
8/14/2019 PCI Express System Architecture_Mindshare Book
118/1103
-- -
- --
- - - -- - - - - - - - ---
- --- - - - - - -- - - - -
- --
-- - - -- -- - - - - - - --- - - - -- - --
- - - -- - - - - - -
- - - - - -
- -- - -
8/14/2019 PCI Express System Architecture_Mindshare Book
119/1103
- -- --
-- -- - - -- - - -- - - - - -- -- - - --- - - - -- - -
- --- -- -- - -
- - - - - - --
-- - - - - -- -- -- - - -
- -- - - --
- -
Requester Completer
MWr
Legend:
MWr = Memory Write Request. No completions for this transaction
8/14/2019 PCI Express System Architecture_Mindshare Book
120/1103
-- -
- --
- - -- - - - - - - - - ---
-- -- - - - -
-- - - - -- - - - - - - -
- -- -
Requester Completer
Msg, MsgD
Legend:
Msg = Message Request without data
MsgD = Message Request with data
8/14/2019 PCI Express System Architecture_Mindshare Book
121/1103
- - -
- - - - -- -- -- - -- - - - -- -- - - - - - -- - - - - - -- - -- - - - -
- - - - -
- - - - - -
-
Processor Processor
Root ComplexDDR
SDRAM
Endpoint Endpoint Endpoint
Endpoint Endpoint
Switch A Switch C
Switch B
FSB
MRd
MRd
MRd CplD
CplD
CplD
Requester:-Step 1: Root Complex (requester)initiates Memory Read Request (MRd)
-Step 4: Root Complex receives CplD
Completer:-Step 2: Endpoint (completer)receives MRd-Step 3: Endpoint returnsCompletion with data (CplD)
MRd
8/14/2019 PCI Express System Architecture_Mindshare Book
122/1103
-- -
- - -
- - - - -- - - - - - - - - - - -- --- - - - -- - -
-
- - - - - - - -
- - -- - - - -- - - -- -- - - - - -- -
- - -- - - - - - --
- -
- - - - - -- - - - - - - -- - -- - - -
8/14/2019 PCI Express System Architecture_Mindshare Book
123/1103
-- -- - -- -- - -- - - - -- -- - -- -- -
-- -- - --- - - -- - -- -- - -- -
-- - -- - - - - -- - - - - -- - -- - - - - - - -- --- -
-
Processor Processor
Root ComplexDDR
SDRAM
Endpoint Endpoint Endpoint
Endpoint
Switch A Switch C
Switch B
FSB
Requester:-Step 1: Endpoint (requester)
initiates Memory Read Request (MRd)
-Step 4: Endpoint receives CplD
MRd
MRd
MRd
Completer:
-Step 2: Root Complex (completer)
receives MRd
-Step 3: Root Complex returns
Completion with data (CplD)
CplD
CplD
CplD
Endpoint
8/14/2019 PCI Express System Architecture_Mindshare Book
124/1103
-- -
-
-- -- -- - - --- - - - - -- -- - - - -- - -
-- - - - -- - - - - - - -
-
Processor Processor
Root ComplexDDR
SDRAM
Endpoint Endpoint Endpoint
EndpointLegacy
Endpoint
Switch A Switch C
Switch B
FSB
IOWr
IOWr
IOWr Cpl
Cpl
Cpl
Requester:
-Step 1: Root Complex (requester)
initiates IO Write Request (IOWr)
-Step 4: Root Complex receives Cpl
Completer:
-Step 2: Endpoint (completer)
receives IOWr
-Step 3: Endpoint returns
Completion without data (Cpl)
8/14/2019 PCI Express System Architecture_Mindshare Book
125/1103
-- -
-- - - - -- - -- - - - - --- --- - - - - - - -
-
Processor Processor
Root ComplexDDR
SDRAM
Endpoint Endpoint Endpoint
Endpoint Endpoint
Switch A Switch C
Switch B
FSB
Requester:
-Step 1: Root Complex (requester)
initiates Memory Write Request (MWr)
MWr
MWr
MWr
Completer:
- Step 2: Endpoint (completer) receives MWr
8/14/2019 PCI Express System Architecture_Mindshare Book
126/1103
-- -
- - - - - - - - - -
-- -
Device Core
PCI Express Core
Logic Interface
Transaction Layer
Data Link Layer
Physical Layer
TX RX
PCI Express Device A
Device Core
PCI Express Core
Logic Interface
Transaction Layer
Data Link Layer
Physical Layer
TX RX
PCI Express Device B
Link
8/14/2019 PCI Express System Architecture_Mindshare Book
127/1103
- -
- - - - - - - --- -- - - - - - - - - - - - - - - - - -
-
- - - - - - - - - - - - - - - - - --
- -- -
- - - -- - -- - - - -- - -- - - - - -
- - -
-- -- - - - - - -- - - - - -- - - - - - - - ---
8/14/2019 PCI Express System Architecture_Mindshare Book
128/1103
-- -
-- - - - --
- -- - -- - - - - - - - - -
- - - - - - - - - - - - - - -
- - -
-
Device Core
PCI Express Core
Logic Interface
Transaction Layer
Data Link Layer
Physical Layer
TX RX
PCI Express Device A
Device Core
PCI Express Core
Logic Interface
Transaction Layer
Data Link Layer
Physical Layer
TX RX
PCI Express Device B
Link
TLP
Transmitted
TLP
Received
8/14/2019 PCI Express System Architecture_Mindshare Book
129/1103
--- - - - - - - - - - - - - - - -- - -- - - - - - - - -- - -- - - - - - - - - -
--
Start EndSequence
NumberLCRCHeader Data
Created by Transaction Layer
Appended by Data Link Layer
Appended by PHY Layer
ECRC
Bit transmit direction
Information in core section of TLP comes
from Software Layer / Device Core
8/14/2019 PCI Express System Architecture_Mindshare Book
130/1103
-- -
- - -
- - - - -
- -
-- - - - - -- - - - -- -- --- - - --
- - - --
---
Start EndSequence
NumberLCRCHeader Data
Stripped by Transaction Layer
Stripped by Data Link Layer
Stripped by PHY Layer
ECRC
Bit receive direction
Information in core section of TLP is
sent to Software Layer / Device Core
8/14/2019 PCI Express System Architecture_Mindshare Book
131/1103
- - -
- - -- -- - -- - - - - - - - -
-- - - - - - - - - - - - - - - - - -
- - - - - - -
-
Device Core
PCI Express Core
Logic Interface
Transaction Layer
Data Link Layer
Physical Layer
TX RX
PCI Express Device A
Device Core
PCI Express Core
Logic Interface
Transaction Layer
Data Link Layer
Physical Layer
TX RX
PCI Express Device B
Link
DLLP
Transmitted
DLLP
Received
8/14/2019 PCI Express System Architecture_Mindshare Book
132/1103
-- -
--- - - - - - - - - - - - - - -- - - - - -
--
---
Start EndDLLP CRC
Data Link Layer
Appended by PHY Layer
Bit transmit direction
Start EndDLLP CRC
Stripped by Data Link Layer
Stripped by PHY Layer
Bit receive direction
8/14/2019 PCI Express System Architecture_Mindshare Book
133/1103
- - -
-- - - - - - - -- - - - - -- - - - - - - - - - - - - - -
- - -- - - -
- - - - - - - -
-
Device Core
PCI Express Core
Logic Interface
Transaction Layer
Data Link Layer
Physical Layer
TX RX
PCI Express Device A
Device Core
PCI Express Core
Logic Interface
Transaction Layer
Data Link Layer
Physical Layer
TX RX
PCI Express Device B
Link
PLP
Transmitted
PLP
Received
8/14/2019 PCI Express System Architecture_Mindshare Book
134/1103
-- -
-- - --- - - - - - - -
-- - -- - -
--- - - - -- - -
- -- - - - -
- -- -- -- - - - - - - - - - - - -- - -- --
- - -- - - -
- - -- -- -- -
COM Identifier Identifier Identifier
8/14/2019 PCI Express System Architecture_Mindshare Book
135/1103
-
- - - -- - -- - - - -- - - --- - - - -
-- - -
Port
Link
Memory, I/O, Configuration R/W Requests or Message Requests or Completions
(Software layer sends / receives address/transaction type/data/message index)Software layer
Transaction layer
Header Data Payload ECRC
Receive
Buffers
per VC
Transmit
Buffers
per VC
Data Link layer
Physical layer
Serial-to-ParallelParallel-to-Serial
Differential ReceiverDifferential Driver
Link PacketStart EndLink PacketStart End
DecodeEncode
Transaction Layer Packet (TLP)
TLP Error
Check
TLP Replay
Buffer
Physical PacketPhysical Packet
Link Packet
Sequence TLP LCRCACK/NAK
DLLPs e.g.
CRC
Header Data Payload ECRC
Transaction Layer Packet (TLP)
Link Packet
Sequence TLP LCRC
De-mux
Mux
Transmit Receive
Flow Control
Virtual Channel
Management
Ordering
LinkTraining
ACK/NAK
DLLPs
CRC
8/14/2019 PCI Express System Architecture_Mindshare Book
136/1103
-- -
- - - - -
- --- - - - -- - - -- - - - - - - - - - - - --- - - --
- -- -- -- -- -- -- -- - -- - --
- - - - - - -- - - --- - - -
- - - -- - - -
- - - -
- - -- - - - -- - - - - - -
-- - -- -- -- - -- -- -- -- - -- - - - -- - - -- - - - -- -- -- - - - - -- - -- - -- --
- - -
-
Header Data Payload ECRC
Transaction Layer Packet (TLP)
8/14/2019 PCI Express System Architecture_Mindshare Book
137/1103
- - - -
- - - - - - - - - - - - - - - - -- -
- - - - - - - - - - -
- - - - - - - - - - -
- - - - - - - - - - - - -- - - - - - -
- - -- - - - -
- - -- - - -
- -
- -- - - -- - - - -- - - - - - - - - - - - - - - - - -
- - - - - -- -
- -- -- - -
8/14/2019 PCI Express System Architecture_Mindshare Book
138/1103
-- -
--
-- - - - - -- - - - - - - - - - - - - - - - -
- - - - - -- - -- - - - - - -- - -- - - - -- - -- - - - - - - --- -
- - -
--
ReceiverTransmitter
Flow Control DLLP (FCx)
TLPVC Buffer
Buffer space
available
8/14/2019 PCI Express System Architecture_Mindshare Book
139/1103
--- - --
-- - - - -- -- - - - -
- - - - - -- - - - - - - -- - - - -- -- - - - - - - -
-- - -
--
Processor Processor
Root ComplexGFX DDRSDRAM
10GbEthernet
PCI Expressto-PCI
PCI
InfiniBandSwitch
SCSI
RAID Disk array
IEEE
1394
Out-of-Box
InfiniBand
SCSIPCI Express
Link
Switch Switch
Switch
PCI ExpressGFX
FSB
SIO
COM1COM2
SIO
COM1COM2
Endpoint
Endpoint
Endpoint Endpoint
Endpoint
Endpoint
10Gb
Ethernet
Endpoint
Add-In
VideoCamera
Fiber
Channel
Slots
Slot
8/14/2019 PCI Express System Architecture_Mindshare Book
140/1103
-- -
- - - -
- - - -- - - - - - - -
- -- -- - - - - -- - - - - -- - -
--- - - - - - - -- - -
- - -- - - - - - - - - - -
-- - - - - --- - - - - - - - -- -
- - -- - - - -
- --
- - -- - - - - - - - - -- - -- - - - - - - -- --
- - -- - - -- - - - -
- -
8/14/2019 PCI Express System Architecture_Mindshare Book
141/1103
- -
- - - - - -- - - - -
- - - - -- -
- -
- - -
-- - - --
-
- - -- - -- - -- - - -- - --
- -
Receiver Device B
VC0
VC1
TC[7:0]
TC[2:0]
maps to
VC0
TC[7:3]
maps to
VC1
Transmitter Device A
Link
Buffers Buffers TC[7:0]
VC0
VC1
VC0
VC1
One physical Link,
multiple virtual paths
8/14/2019 PCI Express System Architecture_Mindshare Book
142/1103
-- -
- - --
-
- - --- - -- - - -- - --
- -- - - - - -- -- - - - - - -- -
--
- - - - - -
- - - -
-
VC0
0
VC1
TC[2:0] to VC0
TC[7:3] to VC1
Link
VC0
VC1
TC[2:0] to VC0
TC[7:3] to VC1
Link
VC0Port
Arb
VC1
Port
Arb
VC
Arb
VC0
VC1
Link
VC0
VC1
8/14/2019 PCI Express System Architecture_Mindshare Book
143/1103
- -- -
- - - - - - - -- -- - - -- - - - -- - -
- -- - - - - - ---
- --
- -- - - - - -- - - - - - - -- - - -- -- - -
-- - -- -- - -- - - - --
- -- - - - - -- -
- - - --- - -- - - - - - - - - -- -- - - - - - -
-- --- - - --
8/14/2019 PCI Express System Architecture_Mindshare Book
144/1103
-- -
- - - -
- - -- - - - -- - ---
- -- -- - - --
- - - - - - -- -- - - -
-
ErrorCheck
Replay
Buffer
Link Packet
Sequence TLP LCRCACK /
NAK
DLLP
De-mux
Mux
From
Transaction Layer
To
Transaction Layer
Data Link Layer
Tx Rx
Link Packet
Sequence TLP LCRCACK /
NAK
DLLP
Link
Device A
Remote
Device B
Tx Rx
Error
Check
Replay
Buffer
LinkPacket
SequenceTLPLCRCACK/
NAK
DLLP
De-mux
Mux
From
TransactionLayer
To
TransactionLayer
DataLinkLayer
TxRx
LinkPacket
SequenceTLPLCRCACK/NAK
DLLP
TxRx
8/14/2019 PCI Express System Architecture_Mindshare Book
145/1103
- -
- - - -- - --- - - -- -
- - - - --- - - -- - - - --
- - -- - - -- - -
- - -
- - -- -
- - - - - - - - - -
- - -- - - -- --- - -- -
- -- - -
- - - - - - - - - - -- - - -- - - -- -
- - - -- - - -
- - - - - - - - - - - - --
- -
8/14/2019 PCI Express System Architecture_Mindshare Book
146/1103
-- -
- - -
-
- - - - - - - - - - - - - - - - -- - - - - --- -- - - - - - - - --
-
- - -- - - - - -- - -
- -- - - - -
Header Data Payload ECRC
Transaction Layer Packet (TLP)
Sequence ID LCRC
DLLP Type CRC
DLLP
AND
Misc.
8/14/2019 PCI Express System Architecture_Mindshare Book
147/1103
- - - - -
- -
- -- - -- - - - -
- - -- -
- - -- - - - - - -
- - -- -
- - - -- - - - - - -
- - - -- - - - - - -- - - -- -- -- - - - - - -
- -
ACK returned for good reception of Request or Completion
NAK returned for error reception of Request or Completion
Requester Completer
1a. Request
Switch
2a. Request
3a. Completion4a. Completion
1b. ACK 2b. ACK
4b. ACK 3b. ACK
8/14/2019 PCI Express System Architecture_Mindshare Book
148/1103
-- -
- -
-- - - - -- - -
- -- - - -
- - - - -- -
- -- - -- - - - -
- - -- - - - - -- - - -- - -
- - - - - - -- - - - -
- - - - -- - - -
- -
Requester Completer
1a Request
Switch
2a Request
1b ACK 2b ACK
ACK returned for good reception of Request or CompletionNAK returned for error reception of Request or Completion
8/14/2019 PCI Express System Architecture_Mindshare Book
149/1103
-
- - - - - --- - - - -
- - - - - - - -- -- - --- -- - - - - - - --- - -
- - - - - - - - -- - - - -- -
- - - --- - --- -
- - - - - - -
- -
- - - - -- - - - - - - - - - - - --
- - - - -- --
- -
- - - - - - - - - - -
8/14/2019 PCI Express System Architecture_Mindshare Book
150/1103
-- -
- - -
- - - - - - - - - - -
- - - - -- - - - -- - - -
--
- - -
- - - - - - - --
- - - - -
-
Header Data Payload ECRC
Transaction Layer Packet (TLP)
Sequence LCRC
DLLP Type CRC
DLLP
Misc.Start End
Start End
1B 2B 1DW 1B1DW0-1024 DW3-4 DW
1B 1B1DW 2B
8/14/2019 PCI Express System Architecture_Mindshare Book
151/1103
- -- - -
- - - - - - - - - - - -- - -
- - - - - --
- - - - - - -- - - - - - - - -
- - -- -- - - - - - - - - - -
- - -- - - - -- - -- - - - - - - -- -- -
- - - -
- - - -
- - - - - - -
- -- - - - - --- - - - - - - -
- - -
-- - - - - --
8/14/2019 PCI Express System Architecture_Mindshare Book
152/1103
-- -
- - -
- - - - - - -- - - - - - - -
- -
- - - - - - - - - - - - - - - -- -- - -- --- - -
- - -
- -
+
-
Receiver
VttZvtt
Transmitter
CTX
CTX
ZTX
ZRX
ZRXZTX
ZTX
ZTX
Link
Transmitter is AC coupled to receiver
DC common mode impedance is 50 Ohms
Differential impedance is 100 Ohms
Coupling capacitor is between 75-200 nF
8/14/2019 PCI Express System Architecture_Mindshare Book
153/1103
- -
- -- -
-- - - - -- - - - - - -
- -- - - - -
-- - - - - - - -- - - -
- -
Send Memory Read Request
Header Data Payload ECRC
Flow Control
Virtual Channel
Management
Ordering
Receive
Buffersper VC
Transmit
Buffers
per VC
Requester
Serial-to-ParallelParallel-to-Serial
Differential ReceiverDifferential Driver
Link PacketStart EndLink PacketStart End
DecodeEncode
Transaction Layer Packet (TLP)
Error
Check
Replay Buffer
Physical PacketPhysical Packet
Link Packet
Sequence TLP LCRC
ACK/NAK
DLLP.
CRC
Header Data Payload ECRC
Transaction Layer Packet (TLP)
Link Packet
Sequence TLP LCRC
PortPort
MRdTLPLink
ACKor NAK
NAK
DLLP
Physical layer
Software layer
Data Link layer
Software layer
Transaction layer Flow Control
Virtual Channel
Management
Ordering
Completer
Receive Memory Read Request
8/14/2019 PCI Express System Architecture_Mindshare Book
154/1103
-- -
- -
- --- - -- -- -- -
- - - - - -
- - --
- - - - - - - - -- - - - -
- - - --
- - - -- -- - --
- - - - - -- - - -
- - - -- - - -- - - - - - - -
- - - -- -
- - - -- - - - -- - - -- - - - - - - - -
-- - - - -
- - - -- - - - - --
8/14/2019 PCI Express System Architecture_Mindshare Book
155/1103
- - -
--
-
--- - - -- -
- - - - -- - - - - - - - - -- ---
- -
-
Header Data Payload ECRC
ReceiveBuffers
per VC
Serial-to-Parallel
Differential Receiver
Link PacketStart End
Decode
Transaction Layer Packet (TLP)
Error
Check
Physical Packet
Link Packet
Sequence TLP LCRC
ACK/NAK
DLLP
CRC
Port
LinkACK or NAK
Physical layer
Software layer
Data Link layer
Software layer
Transaction layerFlow Control
Virtual Channel
ManagementOrdering
Receive Completion with Data) Send Completion with Data
Flow Control
Virtual Channel
ManagementOrdering
Transmit
Buffersper VC
Parallel-to-Serial
Differential Driver
Link PacketStart End
Encode
Replay Buffer
Physical Packet
Header Data Payload ECRC
Transaction Layer Packet (TLP)
Link Packet
Sequence TLP LCRC
Port
NAK
DLLP
CplDTLP
Requester Completer
8/14/2019 PCI Express System Architecture_Mindshare Book
156/1103
-- -
- -- -
- - - - - - - - - - - - - - - - - -- ---
- - - - - - - - -
- - - - - - - - -- - - - - - - - -- -
- - - - -- -- - -- - - - - - -- - - -
- -
- -- - - -- - - - - - - - - - --
- - -- - - - - - - -- - - - - - -
- - -
- -- -- - - - -
8/14/2019 PCI Express System Architecture_Mindshare Book
157/1103
- -
- -
-- -- - - - -- --
- - - - - - - -- -
- - - -
- - - - - - - - - - --
- - -- - - -
-- -
- - - -- - - -
- - - - - - - - -
-- - -
--
--
8/14/2019 PCI Express System Architecture_Mindshare Book
158/1103
-- -
- -
- - -- --- - - - -
- - - - - - - - - - -
- - -
- - - -- - -- - - -
- -
- -- - - - -- - - - - - - --
- - - - -- --- - - - - -- -
8/14/2019 PCI Express System Architecture_Mindshare Book
159/1103
-
8/14/2019 PCI Express System Architecture_Mindshare Book
160/1103
8/14/2019 PCI Express System Architecture_Mindshare Book
161/1103
-- - - -
- -- - - -- - -
- - -- - - - -- - - - - --
-
- -- - -- - -- - -- - -- - - -- - - -
- -- -- - - - - -- - -- -- - - - - -- - --
- --- - - - - - - - - - -
8/14/2019 PCI Express System Architecture_Mindshare Book
162/1103
-- -
-- - - - - - - - -- -
-- - ---
8/14/2019 PCI Express System Architecture_Mindshare Book
163/1103
-- - -
- - -- ---
- - - - - -- - - - --
- -- - -
- --
- -
-- - -
- - - - - - -- - -- - - - - - - - - -- - - - - - - -
- --
- - -- - -- - - -- - -- - - - - - -- - - -- - - -
- ---
- - - - -
- - - - - --- - - -- - - - -- - ---
8/14/2019 PCI Express System Architecture_Mindshare Book
164/1103
-- -
- -
-- -
- -
-- - - - - -- - - - - --