[Gnash-commit] /srv/bzr/gnash/rtmp r9687: DiskStream class now usable.

[rob]

------------------------------------------------------------
revno: 9687
committer: address@hidden
branch nick: rtmp
timestamp: Mon 2008-11-03 11:02:46 -0700
message:
  DiskStream class now usable.
added:
  cygnal/testsuite/cygnal.all/test_diskstream.cpp
modified:
  cygnal/diskstream.cpp
  cygnal/diskstream.h

=== modified file 'cygnal/diskstream.cpp'
--- a/cygnal/diskstream.cpp     2008-11-01 14:51:14 +0000
+++ b/cygnal/diskstream.cpp     2008-11-03 18:02:46 +0000
@@ -48,12 +48,21 @@
 
 namespace cygnal {
 
+/// \def _SC_PAGESIZE
+///    This isn't set on all systems, but is used to get the page
+///    size used for memory allocations.
+#ifndef _SC_PAGESIZE
+#define _SC_PAGESIZE 8
+#endif
+
 DiskStream::DiskStream()
     : _bytes(0),
       _filefd(0),
       _netfd(0),
+      _dataptr(0),
       _filesize(0),
-      _chunksize(0)
+      _pagesize(0),
+      _offset(0)
 {
 //    GNASH_REPORT_FUNCTION;
 }
@@ -62,8 +71,10 @@
     : _bytes(0),
       _filefd(0),
       _netfd(0),
+      _dataptr(0),
       _filesize(0),
-      _chunksize(0)
+      _pagesize(0),
+      _offset(0)
 {
 //    GNASH_REPORT_FUNCTION;
     _filespec = str;
@@ -73,52 +84,126 @@
     : _bytes(0),
       _filefd(0),
       _filespec(0),
+      _dataptr(0),
       _filesize(0),
-      _chunksize(0)
+      _pagesize(0),
+      _offset(0)
 {
 //    GNASH_REPORT_FUNCTION;
     _netfd = netfd;
     _filespec = str;
 }
 
-#if 0
-
-// Load a chunk of the file into memory
-size_t
-DiskStream::loadChunk(size_t size)
-{
-//    GNASH_REPORT_FUNCTION;
-
-#ifdef HAVE_SYSCONF
-    long pageSize = sysconf(_SC_PAGESIZE);
-    if (size % pageSize) {
-        size += pageSize - size % pageSize;
-//      log_debug("Adjusting segment size to %d to be page aligned.\n", _size);
-    }
-#endif
-
-#if 1
-    
-    if (_filefd) {
-       _dataptr = static_cast<unsigned char *>(mmap(0, _chunksize,
-                                                    PROT_READ, MAP_SHARED, 
_filefd, 0));
-    } else {
-       log_error (_("Couldn't load file %s"), filespec);
-       return false;
-    }
-    
-    if (_seekptr == MAP_FAILED) {
+DiskStream::~DiskStream() {
+//    GNASH_REPORT_FUNCTION;
+    if (_filefd) {
+        ::close(_filefd);
+    }
+    if (_netfd) {
+       ::close(_netfd);
+    }
+}
+
+    /// \brief close the open disk file and stream.
+void
+DiskStream::close()
+{
+//    GNASH_REPORT_FUNCTION;
+    _filesize = 0;
+    _offset = 0;
+    if ((_dataptr != MAP_FAILED) && (_dataptr != 0)) {
+       munmap(_dataptr, _pagesize);
+    }
+    _dataptr = 0;
+    _filespec.clear();
+}
+
+void
+DiskStream::dump()
+{
+//    GNASH_REPORT_FUNCTION;
+    //state_e     _state;
+    cerr << "Bytes read is is " << _bytes << endl;
+    cerr << "Disk file descriptor is " << _filefd << endl;
+    cerr << "Network file descritor is " << _netfd << endl;
+    cerr << "Filespec is " << _filespec << endl;
+//    gnash::Statistics  _statistics;
+//     unsigned char *_dataptr;
+//     unsigned char *_seekptr;
+    cerr << "File size is " <<  _filesize << endl;
+    cerr << "Memory Page size is " << _pagesize << endl;
+
+    _que.dump();
+}
+
+/// \brief Load a chunk (pagesize) of the file into memory.
+///    This loads a pagesize of the disk file into memory. We read
+///    the file this way as it is faster and takes less resources
+///    than read(), which add buffering we don't need.
+///
+boost::uint8_t *
+DiskStream::loadChunk(off_t offset)
+{
+//    GNASH_REPORT_FUNCTION;
+
+    /// If the data pointer is left from a failed mmap, don't do
+    /// anything.
+    if (_dataptr ==  MAP_FAILED) {
+       log_error("Bad pointer to memory for file %s!", _filespec);
+       return 0;
+    }
+
+    /// We only map pages of pagesize, so if the offset is smaller
+    /// than that, don't use it.
+    if (static_cast<size_t>(offset) < _pagesize) {
+       _offset = 0;
+//     log_debug("Loading first segment");
+    } else {
+       if (offset % _pagesize) {
+           // calculate the number of pages
+           int pages = ((offset - (offset % _pagesize)) / _pagesize);
+           _offset = pages * _pagesize;
+//         log_debug("Adjusting offset from %d to %d so it's page aligned.",
+//                   offset, _offset);
+       }
+    }
+    
+    if (_filefd) {
+    /// If the data pointer is legit, then we need to unmap that page
+    /// to mmap() a new one. If we're still in the current mapped
+    /// page, then just return the existing data pointer.
+    if (_dataptr != 0) {
+       // If the offset is less than what we already mmapped, we
+       boost::uint32_t diff = reinterpret_cast<boost::uint32_t>(_dataptr + 
_offset);
+       if (diff < _pagesize) {
+           return _dataptr + _offset;
+           // unmap the old data before allocating a new chunk
+       } else {
+           munmap(_dataptr, _pagesize);
+           _dataptr = 0;
+       }
+    }
+
+       _dataptr = static_cast<unsigned char *>(mmap(0, _pagesize,
+                                       PROT_READ, MAP_SHARED, _filefd, 
_offset));
+       offset = (offset % _pagesize);
+    } else {
+       log_error (_("Couldn't load file %s"), _filespec);
+       return 0;
+    }
+    
+    if (_dataptr == MAP_FAILED) {
        log_error (_("Couldn't map file %s into memory: %s"),
-                  filespec, strerror(errno));
-       return false;
-    } else {       
-       log_debug (_("File %s mapped to: %p"), filespec,
-                  (void *)_dataptr);
-       _seekptr = _dataptr;
+                  _filespec, strerror(errno));
+       return 0;
+    } else {
+//     log_debug (_("File %s mapped to: %p"), _filespec, (void *)_dataptr);
+       _seekptr = _dataptr + offset;
        _state = OPEN;
-       return true;
     }
-#else
+    
+    return _seekptr;
+#if 0
     do {
        boost::shared_ptr<amf::Buffer> buf(new amf::Buffer);
        ret = read(filefd, buf->reference(), buf->size());
@@ -140,16 +225,6 @@
 #endif
 }
 
-DiskStream::~DiskStream() {
-//    GNASH_REPORT_FUNCTION;
-    if (_filefd) {
-        close(_filefd);
-    }
-    if (_netfd) {
-       ::close(_netfd);
-    }
-}
-
 bool
 DiskStream::open(const string &filespec) {
 //    GNASH_REPORT_FUNCTION;
@@ -169,15 +244,16 @@
 
 bool
 DiskStream::open(const string &filespec, int netfd, Statistics &statistics) {
-    GNASH_REPORT_FUNCTION;
+//    GNASH_REPORT_FUNCTION;
 
     struct stat st;
     
     _netfd = netfd;
     _statistics = statistics;
+    _filespec = filespec;
 
     log_debug("Trying to open %s", filespec);
-
+    
     if (stat(filespec.c_str(), &st) == 0) {
        _filesize = st.st_size;
        boost::mutex::scoped_lock lock(io_mutex);
@@ -187,10 +263,25 @@
     } else {
        log_error (_("File %s doesn't exist"), filespec);
     }
+
+    /// \brief get the pagesize and cache the value
+#ifdef HAVE_SYSCONF
+    _pagesize = sysconf(_SC_PAGESIZE);
+#else
+#error "Need to define the memory page size without sysconf()!"
+#endif
+
+//     // The pagesize is how much of the file to load. As all memory is
+//     // only mapped in multiples of pages, we use that for the default size.
+//     if (_pagesize == 0) {
+//     _pagesize = pageSize;
+//     }
     
     return true;
 }
 
+#if 0
+
 // Stream the file
 bool
 DiskStream::play() {

=== modified file 'cygnal/diskstream.h'
--- a/cygnal/diskstream.h       2008-11-01 14:51:14 +0000
+++ b/cygnal/diskstream.h       2008-11-03 18:02:46 +0000
@@ -24,9 +24,11 @@
 #endif
 
 //#ifdef HAVE_AIO_H
+//#include <aio.h>
+//#endif
 
-#include <aio.h>
 #include <string>
+#include <iostream> 
 
 #include "cque.h"
 #include "statistics.h"
@@ -51,6 +53,9 @@
     DiskStream(const std::string &filespec);
     DiskStream(const std::string &filespec, int netfd);
     ~DiskStream();
+
+    /// \brief close the open disk file and stream.
+    void close();
     
     bool open(const std::string &filespec);
     bool open(const std::string &filespec, int netfd);
@@ -78,10 +83,18 @@
     // Stream a single "real-time" source.
     bool multicast(const std::string &filespec);
 
-    // Load a chunk of the file into memory
-    size_t loadChunk(size_t size);
-    size_t loadChunk() { return loadChunk(_chunksize); };
+    /// \brief Load a chunk of the file into memory
+    ///                This offset must be a multipe of the pagesize.
+    boost::uint8_t *loadChunk(off_t size);
+    boost::uint8_t *loadChunk() { return loadChunk(_offset); };
+
+    size_t getPagesize() { return _pagesize; };
+    void setPagesize(size_t size) { _pagesize = size; };
     
+    void dump();
+//    friend std::ostream& operator<< (std::ostream &os, const DiskStream &ds);
+
+    boost::uint8_t *get() { return _dataptr; };
 private:
     state_e     _state;
     int         _bytes;
@@ -89,11 +102,12 @@
     int         _netfd;
     std::string _filespec;
     gnash::Statistics  _statistics;
-//     unsigned char *_dataptr;
-//     unsigned char *_seekptr;
+    boost::uint8_t *_dataptr;
+    boost::uint8_t *_seekptr;
     size_t     _filesize;
-    size_t     _chunksize;
-    struct aiocb _aio_control_block;
+    size_t     _pagesize;
+    off_t      _offset;
+//    struct aiocb _aio_control_block;
     gnash::CQue _que;
 };
  

=== added file 'cygnal/testsuite/cygnal.all/test_diskstream.cpp'
--- a/cygnal/testsuite/cygnal.all/test_diskstream.cpp   1970-01-01 00:00:00 
+0000
+++ b/cygnal/testsuite/cygnal.all/test_diskstream.cpp   2008-11-03 18:02:46 
+0000
@@ -0,0 +1,244 @@
+// 
+//   Copyright (C) 2008 Free Software Foundation, Inc.
+// 
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 3 of the License, or
+// (at your option) any later version.
+// 
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+// 
+// You should have received a copy of the GNU General Public License
+// along with this program; if not, write to the Free Software
+// Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+
+#ifdef HAVE_CONFIG_H
+#include "gnashconfig.h"
+#endif
+
+#ifdef HAVE_DEJAGNU_H
+
+#include <string>
+#include <sys/types.h>
+#include <sys/stat.h>
+
+#if !defined(HAVE_WINSOCK_H) && !defined(__riscos__) && !defined(__OS2__)
+#include <sys/mman.h>
+#endif
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <log.h>
+#include <iostream>
+#include <string>
+#include "as_value.h"
+#include "as_object.h"
+
+#include "dejagnu.h"
+#include "as_object.h"
+#include "arg_parser.h"
+#include "buffer.h"
+#include "diskstream.h"
+
+using namespace amf;
+using namespace gnash;
+using namespace cygnal;
+using namespace std;
+
+static void usage (void);
+
+// Prototypes for test cases
+static void test();
+static void create_file(const std::string &, size_t);
+
+// Enable the display of memory allocation and timing data
+static bool memdebug = false;
+
+TestState runtest;
+LogFile& dbglogfile = LogFile::getDefaultInstance();
+RcInitFile& rcfile = RcInitFile::getDefaultInstance();
+static bool dump = false;
+
+int
+main(int argc, char *argv[])
+{    const Arg_parser::Option opts[] =
+        {
+            { 'h', "help",          Arg_parser::no  },
+            { 'v', "verbose",       Arg_parser::no  },
+            { 'd', "dump",          Arg_parser::no  },
+        };
+    
+    Arg_parser parser(argc, argv, opts);
+    if( ! parser.error().empty() ) {
+        cout << parser.error() << endl;
+        exit(EXIT_FAILURE);
+    }
+    
+    for( int i = 0; i < parser.arguments(); ++i ) {
+        const int code = parser.code(i);
+        try {
+            switch( code ) {
+              case 'h':
+                  usage ();
+                  exit(EXIT_SUCCESS);
+              case 'v':
+                  dbglogfile.setVerbosity();
+                  // This happens once per 'v' flag 
+                  log_debug(_("Verbose output turned on"));
+                  break;
+              case 'd':
+                  dump= true;
+                  break;
+           }
+        }
+        
+        catch (Arg_parser::ArgParserException &e) {
+            cerr << _("Error parsing command line options: ") << e.what() << 
endl;
+            cerr << _("This is a Gnash bug.") << endl;
+        }
+    }
+
+    // run the tests
+    test();
+}
+
+void
+test()
+{
+    DiskStream ds;
+
+    // Create an array of printable ASCII characters
+    int range = '~' - '!';
+    char *buf = new char[range];
+    for (int j=0; j<range; j++) {
+        buf[j] = '!' + j;
+    }
+
+    create_file("outbuf.raw", 500);    
+    ds.open("outbuf.raw");
+
+    // ptr should be the base address of the memory plus the offset
+    boost::uint8_t *ptr = ds.loadChunk(48);
+    boost::uint8_t *dsptr = ds.get(); // cache the initial base address
+    
+    if ((ds.get() == MAP_FAILED) || (ds.get() == 0)) {
+        runtest.unresolved("loadChunk(48)");
+    } else {
+        if ((memcmp(ds.get(), buf, 48) == 0)
+            && (memcmp(ptr, buf+48, range-48) == 0)) {
+            runtest.pass("loadChunk(48)");
+        } else {
+            runtest.fail("loadChunk(48)");
+        }
+    }
+
+    // as the offset is less than the memory pagesize, the pointer
+    // should be the same as before, as it points to data in the
+    // current segment. The temporary pointer should point to the
+    // appropriate place in memory page.
+    ptr = ds.loadChunk(128);
+    if ((ds.get() == MAP_FAILED) || (ds.get() == 0)) {
+        runtest.unresolved("loadChunk(128)");
+    } else {
+        if ((memcmp(ds.get(), buf, range) == 0)
+            && (dsptr == ds.get())
+            && (memcmp(ptr, buf+(128-range), 128-range) == 0)) {
+            runtest.pass("loadChunk(128)");
+        } else {
+            runtest.fail("loadChunk(128)");
+        }
+    }
+
+    // close the currently opened file
+    ds.close();
+    if (ds.get() == 0) {
+        runtest.pass("close()");
+    } else {
+        runtest.fail("close()");
+    }
+
+    // Create a bigger file that's larger than the page size
+    create_file("outbuf2.raw", 12000);
+    ds.open("outbuf2.raw");
+    ptr = ds.loadChunk(6789);
+    if ((ds.get() == MAP_FAILED) || (ds.get() == 0)) {
+        runtest.unresolved("loadChunk(6789)");
+    } else {
+        if ((memcmp(ds.get(), buf+4, range-4) == 0)
+            && (memcmp(ptr, buf, range) == 0)
+            && (dsptr == ds.get())) {
+            runtest.pass("loadChunk(6789)");
+        } else {
+            runtest.fail("loadChunk(6789)");
+        }
+    }
+    
+//    ds.dump();
+
+    delete[] buf;
+    
+    unlink("outbuf.raw");
+    unlink("outbuf2.raw");
+}
+
+/// \brief create a test file to read in later. This lets us create
+/// files of arbitrary sizes.
+void
+create_file(const std::string &filespec, size_t size)
+{
+    // Open an output file
+//    cerr << "Creating a test file.;
+    
+    int fd = open(filespec.c_str(), O_WRONLY|O_CREAT|O_TRUNC, S_IRWXU);
+    if (fd < 0) {
+        perror("open");
+    }
+
+    // Create an array of printable ASCII characters
+    int range = '~' - '!';
+    char *buf = new char[range];
+    for (int j=0; j<range; j++) {
+        buf[j] = '!' + j;
+    }
+    
+    int total = 0;
+    int ret = 0;
+    for (size_t i=0; i<size; i+=range) {
+        if ((size - total) < range) {
+            ret = write(fd, buf, (size - total));
+        } else {
+            ret = write(fd, buf, range);
+        }
+        total += ret;
+     }
+
+    delete[] buf;
+    close(fd);
+}
+
+static void
+usage (void)
+{
+    cerr << "This program tests diskstream support in the cygnal library." << 
endl
+         << endl
+         << _("Usage: test_diskstream [options...]") << endl
+         << _("  -h,  --help          Print this help and exit") << endl
+         << _("  -v,  --verbose       Output verbose debug info") << endl
+         << _("  -d,  --dump          Dump data structures") << endl
+         << endl;
+}
+
+#else
+
+int
+main(int /*argc*/, char /* *argv[]*/)
+{
+  // nop
+  return 0;  
+}
+
+#endif
+