[linux-dvb] Re: Network: a new hope

[Emard <emard@softhome.net>]

HI

Here is a new report regarding the tracking on the data loss.

I have relatively good news:

binary search for the missing packets in ip capture 
did not find missing packets even in the TS stream!
I supposed my DMA is working correctly (I see no skips, 
no stale data).

Then this opens the possibility that network reception is 
working quite well but the data aren't present in the TS, 
because of probaby some packets are send over modem line or 
other TS (or PID).

I still would like to see a clean section demux rewrite, because 
the present algorithm could miss some data but such kind droppings 
should in much less than 10-20% percentage of data which aren't 
present in TS.

Also I miss the feature to set network dvb0_0 with PID=8192 so that
it will accept all PID and route them through the network layer.
(for that we need section rewrite and more buffer space should be 
allocated, perhaps on-deman)

anyway,
...............
the measurement:

TS was captured for one selected PID 451 during 10 seconds of netsystem 
by an userspace program (source attached), with the stdout redirected 
to file:

tsdump > ts.dump

Immediately after that a packet capture was started:

tcpdump -s 0 -i dvb0_0 -w tcp.dump

Then, tcpdump and tsdump were stopped by ctrl-c and in offline mode the lost 
packets are identified by tcp stream reconstruction

tcpflow -r tcp.dump

A file of >100k was chosen, in its filename was its respective
tcp session parameter. This tcp session was then closely inspected with 
ethereal

ethereal tcp.dump

I located neighbouring packets before and after the 'hole' and
readed out their sequence numbers. Each packet was 1360 byte and
in continuous non-overlapping tcp stream, sequence number after each
packet should be increased by its length = 1360.
The neigbouring packets between the missing one had sequence numbers 
that differ 2*1360 bytes, so there was expected sequence number 
at 1*1360 that must be missing.

Calculating back the hex number of the sequence number, I attempted
to find it in TS dump but there wasn't found this sequence number! 
On the other hand, the neighbouring packet's sequence numbers were 
present in the TS stream.

I repeated this quite tedious hex excercise 4 times of capturing,
locating missing packets and searching for expected sequence number 
and couldn't locate missing sequence in TS capture for the single PID
I was capturing.

Emard
#include <stdio.h>
#include <stdlib.h>
#include <sys/ioctl.h>
#include <unistd.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <time.h>
#include <sys/poll.h>
#include <sys/time.h>

#include <linux/dvb/dmx.h>
#include <linux/dvb/frontend.h>
#include <linux/dvb/video.h>

#define BSIZE 188

int pidt[0x2001];


int section_analysis(buf)
{


}

int main(int argc, char **argv)
{
	int fd, ffd, packets=0;
	struct timeval startt;
	struct dmx_pes_filter_params flt; 
	char *search;
	unsigned char buffer[BSIZE];

	fd=open("/dev/dvb/adapter0/dvr0", O_RDONLY);

	ioctl(fd, DMX_SET_BUFFER_SIZE, 1024*1024);
	
	ffd=open("/dev/dvb/adapter0/demux0", O_RDWR);
	if (ffd<0)
	{
		perror("/dev/dvb/adapter0/demux0");
		return -fd;
	}
	flt.pid=0x2000;
	flt.input=DMX_IN_FRONTEND;
	flt.output=DMX_OUT_TS_TAP;
	flt.pes_type=DMX_PES_OTHER;
	flt.flags=0;
	if (ioctl(ffd, DMX_SET_PES_FILTER, &flt)<0)
	{
		perror("DMX_SET_PES_FILTER");
		return;
	}
	if (ioctl(ffd, DMX_SET_BUFFER_SIZE, 65536)<0)
	{
		perror("DMX_SET_BUFFER_SIZE");
		return;
	}
	if (ioctl(ffd, DMX_START, 0)<0)
	{
		perror("DMX_SET_PES_FILTER");
		return;
	}
	
	gettimeofday(&startt, 0);
	
	if (argc>1)
		search=argv[1];
	else
		search=0;

	while (1)
	{
		int pid, r, ok;
		if ((r=read(fd, buffer, 188))<=0)
		{
			perror("read");
			break;
		}
		if (r!=188)
		{
			printf("only read %d\n", r);
			break;
		}
		if (buffer[0]!=0x47)
		{
			continue;
			printf("desync (%x)\n", buffer[0]);
			while (buffer[0]!=0x47)
				read(fd, buffer, 1);
			continue;
		}
		ok=1;
		pid=((((unsigned)buffer[1])<<8)|((unsigned)buffer[2]))&0x1FFF;
		if (search)
		{
			int i, sl=strlen(search);
			ok=0;
			if (pid != 0x1fff)
				for (i=0; i<(188-sl); ++i)
					if (!memcmp(buffer+i, search, sl))
						ok=1;
		}
		if (ok)
		{
			pidt[pid]++;
			pidt[0x2000]++;
		}
		
		packets++;

#if 1
                if(pid == 451)
                  section_analysis(buffer);

                if(pid == 451)
                {
                  int i;

                  printf("%08X: ", packets);
                  for(i = 0; i < 188; i++)
                  {
                    printf("%02X ", buffer[i]);
                  }
                  printf("\n");
                }
#endif
		if (!(packets & 0xFF))
		{
			struct timeval now;
			int diff;
			gettimeofday(&now, 0);
			diff=(now.tv_sec-startt.tv_sec)*1000+(now.tv_usec-startt.tv_usec)/1000;
			if (diff > 1000)
			{
				int pid=0;
				for (pid=0; pid<0x2001; pid++)
				{
					if (pidt[pid])
						printf("%04x %5d p/s %5d kb/s %5d kbit\n", pid, pidt[pid]*1000/diff, pidt[pid]*1000/diff*188/1024, pidt[pid]*8*1000/diff*188/1000);
					pidt[pid]=0;
				}
				printf("-PID--FREQ------BANDWIDTH-BANDWIDTH-\n");
				startt=now;
			}
		}


	}
	
	close(ffd);
	close(fd);
	return 0;
}