Остання активність 1 month ago

GOES Packet Assemble

Версія 3285259d15b696636d344efd1ad3a216ababa29f

packetassemble.py Неформатований
1#!/usr/bin/env python
2
3import sys, struct, os, packetmanager
4
5FRAMESIZE = 892
6M_PDUSIZE = FRAMESIZE - 6
7EXPORTCORRUPT = False
8
9SEQUENCE_FLAG_MAP = {
10 0: "Continued Segment",
11 1: "First Segment",
12 2: "Last Segment",
13 3: "Single Data"
14}
15
16def ParseMSDU(data):
17 o = struct.unpack(">H", data[:2])[0]
18 version = (o & 0xE000) >> 13
19 type = (o & 0x1000) >> 12
20 shf = (o & 0x800) >> 11
21 apid = (o & 0x7FF)
22
23 o = struct.unpack(">H", data[2:4])[0]
24 sequenceflag = (o & 0xC000) >> 14
25 packetnumber = (o & 0x3FFF)
26 packetlength = struct.unpack(">H", data[4:6])[0] -1
27 data = data[6:]
28 return version, type, shf, apid, sequenceflag, packetnumber, packetlength, data
29
30def CalcCRC(data):
31 lsb = 0xFF
32 msb = 0xFF
33 for c in data:
34 x = ord(c) ^ msb
35 x ^= (x >> 4)
36 msb = (lsb ^ (x >> 3) ^ (x << 4)) & 255
37 lsb = (x ^ (x << 5)) & 255
38 return (msb << 8) + lsb
39
40def CheckCRC(data, crc):
41 c = CalcCRC(data)
42 if not c == crc:
43 print " Expected: %s Found %s" %(hex(crc), hex(c))
44 return c == crc
45
46def SavePacket(channelid, packet):
47 global totalCRCErrors
48 global totalSavedPackets
49 try:
50 os.mkdir("channels/%s" %channelid)
51 except:
52 pass
53
54 if packet["apid"] == 2047:
55 print " Fill Packet. Skipping"
56 return
57
58 filename = "channels/%s/%s_%s.lrit" % (channelid, packet["apid"], packet["version"])
59 print "- Saving packet to %s" %filename
60
61 if packet["framesdropped"]:
62 print " WARNING: Some frames has been droped for this packet."
63
64 datasize = len(packet["data"])
65
66 if not datasize - 2 == packet["size"]: # CRC is the latest 2 bytes of the payload
67 print " WARNING: Packet Size does not match! Expected %s Found: %s" %(packet["size"], len(packet["data"]))
68 if datasize - 2 > packet["size"]:
69 datasize = packet["size"] + 2
70 print " WARNING: Trimming data to %s" % datasize
71
72 data = packet["data"][:datasize-2]
73
74 crc = packet["data"][datasize-2:datasize]
75 crc = struct.unpack(">H", crc)[0]
76 crc = CheckCRC(data, crc)
77 if not crc:
78 print " WARNING: CRC does not match!"
79 totalCRCErrors += 1
80
81 if crc or (EXPORTCORRUPT and not crc):
82 firstorsinglepacket = packet["sequenceflag_int"] == 1 or packet["sequenceflag_int"] == 3
83 f = open(filename, "w" if firstorsinglepacket else "a")
84 f.write(data[10:] if firstorsinglepacket else data) # Remove transport layer size
85 f.close()
86 if packet["sequenceflag_int"] == 2 or packet["sequenceflag_int"] == 3:
87 packetmanager.manageFile(filename)
88 totalSavedPackets += 1
89 else:
90 print " Corrupted frame, skipping..."
91
92def CreatePacket(data):
93 while True:
94 if len(data) < 6:
95 return -1, data
96 version, type, shf, apid, sequenceflag, packetnumber, packetlength, data = ParseMSDU(data)
97 pdata = data[:packetlength+2]
98 if apid != 2047:
99 pendingpackets[apid] = {
100 "data": pdata,
101 "version": version,
102 "type": type,
103 "apid": apid,
104 "sequenceflag": SEQUENCE_FLAG_MAP[sequenceflag],
105 "sequenceflag_int": sequenceflag,
106 "packetnumber": packetnumber,
107 "framesdropped": False,
108 "size": packetlength
109 }
110
111 print "- Creating packet %s Size: %s - %s" % (apid, packetlength, SEQUENCE_FLAG_MAP[sequenceflag])
112 else:
113 apid = -1
114
115 if not packetlength+2 == len(data) and packetlength+2 < len(data): # Multiple packets in buffer
116 SavePacket(sys.argv[1], pendingpackets[apid])
117 del pendingpackets[apid]
118 data = data[packetlength+2:]
119 #print " Multiple packets in same buffer. Repeating."
120 else:
121 break
122 return apid, ""
123
124
125if len(sys.argv) < 2:
126 print "Usage: ./channeldecode.py CHANNELID"
127 print "This will open channels/channel_CHANNELID.bin"
128 exit()
129
130filename = "channels/channel_%s.bin" % sys.argv[1]
131
132f = open(filename, "r")
133fsize = os.path.getsize(filename)
134readbytes = 0
135
136pendingpackets = {}
137
138lastFrameNumber = -1
139totalFrameDrops = 0
140totalCRCErrors = 0
141totalSavedPackets = 0
142lastAPID = -1
143buff = ""
144
145while readbytes < fsize:
146 if fsize - readbytes < FRAMESIZE:
147 print " Some bytes at end of file was not enough for filling a frame. Remaining Bytes: %s - Frame Size: %s" % (fsize-readsize, FRAMESIZE)
148 break
149
150 # Read Data
151 data = f.read(FRAMESIZE)
152 versionNumber = (ord(data[0]) & 0xC0) >> 6
153 scid = (ord(data[0]) & 0x3F) << 2 | (ord(data[1]) & 0xC0) >> 6
154 vcid = (ord(data[1]) & 0x3F)
155
156 counter = struct.unpack(">I", data[2:6])[0]
157 counter &= 0xFFFFFF00
158 counter >>= 8
159
160 # Check for dropped Frames
161 if not lastFrameNumber == -1 and not lastFrameNumber+1 == counter:
162 print " Frames dropped: %s" % (counter-lastFrameNumber-1);
163 totalFrameDrops += counter-lastFrameNumber-1;
164 if not lastAPID == -1: # Fill
165 pendingpackets[lastAPID]["data"] += "\x00" * 878
166 pendingpackets[lastAPID]["framesdropped"] = True
167
168
169 #print "SC: %s ID: %s Frame Number: %s" % (scid, vcid, counter)
170
171 # Demux M_PDU
172 data = data[6:] # Strip channel header
173 fhp = struct.unpack(">H", data[:2])[0] & 0x7FF
174 data = data[2:] # Strip M_PDU Header
175 #print " First Packet Header: %s" %fhp
176 #data is now TP_PDU
177 if not fhp == 2047: # Frame Contains a new Packet
178 # Data was incomplete on last FHP and another packet starts here.
179 if lastAPID == -1 and len(buff) > 0:
180 #print " Data was incomplete from last FHP. Parsing packet now"
181 if fhp > 0:
182 buff += data[:fhp]
183 lastAPID, data = CreatePacket(buff)
184 if lastAPID == -1:
185 buff = data
186 else:
187 buff = ""
188
189 if not lastAPID == -1: # We are finishing another packet
190 if fhp > 0:
191 pendingpackets[lastAPID]["data"] += data[:fhp]
192 SavePacket(sys.argv[1], pendingpackets[lastAPID])
193 del pendingpackets[lastAPID]
194 lastAPID = -1
195
196 # Try to create a new packet
197 buff += data[fhp:]
198 lastAPID, data = CreatePacket(buff)
199 if lastAPID == -1:
200 buff = data
201 else:
202 buff = ""
203 else:
204 if len(buff) > 0 and lastAPID == -1:
205 #print " Data was incomplete from last FHP. Parsing packet now"
206 buff += data
207 lastAPID, data = CreatePacket(buff)
208 if lastAPID == -1:
209 buff = data
210 else:
211 buff = ""
212 elif len(buff) > 0:
213 print " PROBLEM!"
214 elif lastAPID == -1:
215 buff += data
216 lastAPID, data = CreatePacket(buff)
217 if lastAPID == -1:
218 buff = data
219 else:
220 buff = ""
221 else:
222 #print " Appending %s bytes to %s" % (lastAPID, len(data))
223 pendingpackets[lastAPID]["data"] += data
224
225
226 lastFrameNumber = counter
227 readbytes += FRAMESIZE
228
229# One packet can be still in pending packets
230for i in pendingpackets.keys():
231 SavePacket(sys.argv[1], pendingpackets[lastAPID])
232
233print "\n\nReport:"
234print "\tTotal Frames Dropped: %s" %totalFrameDrops
235print "\tTotal Saved Packets: %s" %totalSavedPackets
236print "\tTotal Packet CRC Fails: %s" %totalCRCErrors
237
238f.close()
packetmanager.py Неформатований
1#!/usr/bin/env python
2import os, struct
3
4def manageFile(filename):
5 f = open(filename, "r")
6
7 try:
8 type, filetypecode, headerlength, datalength = readHeader(f)
9 except:
10 print " Header 0 is corrupted for file %s" %filename
11 return
12
13 newfilename = filename
14 while f.tell() < headerlength:
15 data = readHeader(f)
16 if data[0] == 4:
17 print " Filename is %s" % data[1]
18 newfilename = data[1]
19 break
20 f.close()
21 if filename != newfilename:
22 print " Renaming %s to %s/%s" %(filename, os.path.dirname(filename), newfilename)
23 os.rename(filename, "%s/%s" %(os.path.dirname(filename), newfilename))
24 else:
25 print " Couldn't find name in %s" %filename
26
27def readHeader(f):
28 global t
29 type = ord(f.read(1))
30 size = f.read(2)
31 size = struct.unpack(">H", size)[0]
32 data = f.read(size-3)
33
34 if type == 0:
35 filetypecode, headerlength, datalength = struct.unpack(">BIQ", data)
36 return type, filetypecode, headerlength, datalength
37 #print "Header type: %s (%s) File Type Code: %s Header Length %s Data Field Length: %s" %(type, HEADERTYPE_MAP[type], filetypecode, headerlength, datalength)
38 elif type == 1:
39 bitsperpixel, columns, lines, compression = struct.unpack(">BHHB", data)
40 #print "Image Structure Header: "
41 #print " Bits Per Pixel: %s" %bitsperpixel
42 #print " Columns: %s" %columns
43 #print " Lines: %s" %lines
44 #print " Compression: %s" %compression
45 return type, bitsperpixel, columns, lines, compression
46
47 elif type == 2:
48 projname, cfac, lfac, coff, loff = struct.unpack(">32sIIII", data)
49 #print "Image Navigation Record"
50 #print " Projection Name: %s" %projname
51 #print " Column Scaling Factor: %s" %cfac
52 #print " Line Scaling Factor: %s" %lfac
53 #print " Column Offset: %s" %coff
54 #print " Line Offset: %s" %loff
55 return type, projname, cfac, lfac, coff, loff
56
57 elif type == 3:
58 #print "Image Data Function Record"
59 #print " Data: {HIDDEN}"
60 ##print " Data: \n%s" %data
61 return type, data
62
63 elif type == 4:
64 #print "Annotation Record"
65 #print " Data: %s" %data
66 return type, data
67
68 elif type == 5:
69 #print "Timestamp Record"
70 days, ms = struct.unpack(">HI", data[1:])
71 #print " Delta from 1 January 1958"
72 #print " Days: %s" %days
73 #print " Miliseconds: %s" %ms
74 return type, days, ms
75
76 elif type == 6:
77 #print "Ancillary Text"
78 #print " Data: %s" %data
79 return type, data
80
81 elif type == 7:
82 #print "Key Header"
83 #print " Data: %s" %data
84 return type, data
85
86 elif type == 128:
87 imageid, sequence, startcol, startline, maxseg, maxcol, maxrow = struct.unpack(">7H", data)
88 #print "Segment Identification Header"
89 #print " Image Id: %s" %imageid
90 #print " Sequence: %s" %sequence
91 #print " Start Column: %s" %startcol
92 #print " Start Line: %s" %startline
93 #print " Number of Segments: %s" %maxseg
94 #print " Width: %s" %maxcol
95 #print " Height: %s" %maxrow
96 return type, imageid, sequence, startcol, startline, maxseg, maxcol, maxrow
97
98 elif type == 129:
99 #print "NOAA Specific Header"
100 signature, productId, productSubId, parameter, compression = struct.unpack(">4sHHHB", data)
101 #print " Signature: %s" %signature
102 #print " Product ID: %s" %productId
103 #print " Product SubId: %s" %productSubId
104 #print " Parameter: %s" %parameter
105 #print " Compression: %s" %compression
106 return type, signature, productId, productSubId, parameter, compression
107
108 elif type == 130:
109 #print "Header Structured Record"
110 #print " Data: %s" % data
111 return type, data
112
113 elif type == 131:
114 #print "Rice Compression Record"
115 flags, pixel, line = struct.unpack(">HBB", data)
116 #print " Flags: %s" %flags
117 #print " Pixel: %s" %pixel
118 #print " Line: %s" %line
119 return type, flags, pixel, line
120
121 else:
122 #print "Type not mapped: %s" % type
123 return type