def read_fastq(filename):
    return a stream of FASTQ entries, handling gzipped and empty files
    if not filename:
        return itertools.cycle((None,))
    if filename == "-":
        filename_fh = sys.stdin
    elif filename.endswith('gz'):
        if is_python3():
            filename_fh = gzip.open(filename, mode='rt')
        else:
            filename_fh = BufferedReader(gzip.open(filename, mode='rt'))
    else:
        filename_fh = open(filename)
    return stream_fastq(filename_fh) 

def test_fetch_files_function(tmpdir):
    # Setup and run three experiments
    root = tmpdir.strpath
    run_test_experiment(exp_name="experiment 1 alpha", exp_id="1234", root_dir=root)
    run_test_experiment(exp_name="experiment 2 beta", exp_id="5678", root_dir=root)
    run_test_experiment(
        exp_name="experiment 3 alpha beta", exp_id="9990", root_dir=root
    tinydb_reader = TinyDbReader(root)
    res = tinydb_reader.fetch_files(indices=0)
    assert len(res) == 1
    assert list(res[0]["artifacts"].keys()) == ["about"]
    assert isinstance(res[0]["artifacts"]["about"], io.BufferedReader)
    assert res[0]["date"] == datetime.datetime(1999, 5, 4, 3, 2, 1)
    assert res[0]["exp_id"] == "1234"
    assert res[0]["exp_name"] == "experiment 1 alpha"
    assert list(res[0]["resources"].keys()) == ["sacred/__init__.py"]
    assert isinstance(res[0]["resources"]["sacred/__init__.py"], io.BufferedReader)
    assert list(res[0]["sources"].keys()) == ["setup.py"]
    assert isinstance(res[0]["sources"]["setup.py"], io.BufferedReader) 

def test_invalid_schema_resource(self, tmpdir, caplog, schema):
        class FakeProvider(object):
            def get_resource_stream(self, pkg, rsc):
                return io.BufferedReader(io.BytesIO(schema))
        # pkg_resources.resource_stream() cannot be mocked directly
        # Instead mock the module-level function it calls.
        (flexmock(pkg_resources)
            .should_receive('get_provider')
            .and_return(FakeProvider()))
        filename = os.path.join(str(tmpdir), 'config.yaml')
        with open(filename, 'w'):
        tasker, workflow = self.prepare()
        plugin = ReactorConfigPlugin(tasker, workflow, config_path=str(tmpdir))
        with caplog.at_level(logging.ERROR), pytest.raises(Exception):
            plugin.run()
        captured_errs = [x.message for x in caplog.records]
        assert any("cannot validate" in x for x in captured_errs) 

def from_fileobj(cls, fileobj, byteorder="big"):
        """Load an nbt file from a proper file object.
        The method is used by the :func:`load` helper function when the
        ``gzipped`` keyword-only argument is not specified explicitly.
        Arguments:
            fileobj:
                Can be either a standard ``io.BufferedReader`` for
                uncompressed nbt or a ``gzip.GzipFile`` for gzipped nbt
                data. The function simply calls the inherited
                :meth:`nbtlib.tag.Compound.parse` classmethod and sets the
                :attr:`filename` and :attr:`gzipped` attributes depending
                on the argument.
            byteorder:
                Can be either ``"big"`` or ``"little"``. The argument is
                forwarded to :meth:`nbtlib.tag.Compound.parse`.
        self = cls.parse(fileobj, byteorder)
        self.filename = getattr(fileobj, "name", self.filename)
        self.gzipped = isinstance(fileobj, gzip.GzipFile)
        self.byteorder = byteorder
        return self 

def get_SHA1(file_value: io.BufferedReader):
    Get SHA1 hash of the file, directly in memory
    TODO : Fix the input type
    :param file_value: A file to compute the SHA-1
    :return: the SHA-1 of the file in memory
    h = hashlib.sha1()
    b = bytearray(128 * 1024)
    mv = memoryview(b)
    for n in iter(lambda: file_value.readinto(mv), 0):
        h.update(mv[:n])
    return h.hexdigest() 

def __init__(self, course_file: BufferedReader = None, dryrun=False, debug=False, helm_args=None, continue_on_error=False, create_namespace=True):
        self.config = Config()
        self.results = ReckonerInstallResults()
        self.config.dryrun = dryrun
        self.config.debug = debug
        self.config.helm_args = helm_args
        self.config.continue_on_error = continue_on_error
        self.config.create_namespace = create_namespace
        if course_file:
            self.config.course_path = course_file.name
        if self.config.debug:
            logging.warn("The --debug flag will be deprecated.  Please use --helm-args or --dry-run instead.")
        if self.config.helm_args:
            logging.warn("Specifying --helm-args on the cli will override helm_args in the course file.")
        self.course = Course(course_file) 

def idzip_compression(path, output):
        '''Compress a file using  idzip, a gzip-compatible format with random access support.
        if output is None:
            output = '-'
        with click.open_file(output, mode='wb') as outfh:
            writer = _compression.GzipFile(fileobj=outfh, mode='wb')
            with click.open_file(path, 'rb') as infh:
                    infh_wrap = io.BufferedReader(infh)
                    header = infh_wrap.peek(2)
                    if _compression.starts_with_gz_magic(header):
                        click.echo("Detected gzip input file", err=True)
                        infh_wrap = _compression.GzipFile(fileobj=infh_wrap)
                except AttributeError:
                    infh_wrap = infh
                buffer_size = _compression.WRITE_BUFFER_SIZE
                chunk = infh_wrap.read(buffer_size)
                while chunk:
                    writer.write(chunk)
                    chunk = infh_wrap.read(buffer_size)
            writer.close() 

def get_pricing_data(service_code, filters=None, max_cache_age_days=30):
    from ... import config
    if filters is None:
        filters = [("location", region_name(clients.ec2.meta.region_name))]
    get_products_args = dict(ServiceCode=service_code,
                             Filters=[dict(Type="TERM_MATCH", Field=k, Value=v) for k, v in filters])
    cache_key = hashlib.sha256(json.dumps(get_products_args, sort_keys=True).encode()).hexdigest()[:32]
    service_code_filename = os.path.join(config.user_config_dir, "pricing_cache_{}.json.gz".format(cache_key))
        cache_date = datetime.fromtimestamp(os.path.getmtime(service_code_filename))
        if cache_date < datetime.now() - timedelta(days=max_cache_age_days):
            raise Exception("Cache is too old, discard")
        with gzip.open(service_code_filename) as gz_fh:
            with io.BufferedReader(gz_fh) as buf_fh:
                pricing_data = json.loads(buf_fh.read().decode())
    except Exception:
        logger.info("Fetching pricing data for %s", service_code)
        client = boto3.client("pricing", region_name="us-east-1")
        pricing_data = [json.loads(p) for p in paginate(client.get_paginator("get_products"), **get_products_args)]
            with gzip.open(service_code_filename, "w") as fh:
                fh.write(json.dumps(pricing_data).encode())
        except Exception as e:
            print(e, file=sys.stderr)
    return pricing_data 

def test_read_byte_file(smb_share):
    file_path = "%s\\%s" % (smb_share, "file.txt")
    file_contents = b"\x00\x01\x02\x03"
    expected = "[NtStatus 0xc0000034] No such file or directory"
    with pytest.raises(SMBOSError, match=re.escape(expected)):
        smbclient.open_file(file_path, mode='rb')
    with smbclient.open_file(file_path, mode='wb') as fd:
        fd.write(file_contents)
    with smbclient.open_file(file_path, mode='rb') as fd:
        assert isinstance(fd, io.BufferedReader)
        assert fd.closed is False
        assert fd.name == file_path
        actual = fd.read()
        assert actual == file_contents
        actual = fd.read()
        assert actual == b""
        fd.seek(0)
        actual = fd.read()
        assert actual == file_contents
        with pytest.raises(IOError):
            fd.write(b"Fail")
    assert fd.closed 

def extractfile(self, member):
        """Extract a member from the archive as a file object. `member' may be
           a filename or a TarInfo object. If `member' is a regular file or a
           link, an io.BufferedReader object is returned. Otherwise, None is
           returned.
        self._check("r")
        if isinstance(member, str):
            tarinfo = self.getmember(member)
        else:
            tarinfo = member
        if tarinfo.isreg() or tarinfo.type not in SUPPORTED_TYPES:
            # Members with unknown types are treated as regular files.
            return self.fileobject(self, tarinfo)
        elif tarinfo.islnk() or tarinfo.issym():
            if isinstance(self.fileobj, _Stream):
                # A small but ugly workaround for the case that someone tries
                # to extract a (sym)link as a file-object from a non-seekable
                # stream of tar blocks.
                raise StreamError("cannot extract (sym)link as file object")
            else:
                # A (sym)link's file object is its target's file object.
                return self.extractfile(self._find_link_target(tarinfo))
        else:
            # If there's no data associated with the member (directory, chrdev,
            # blkdev, etc.), return None instead of a file object.
            return None 

def backport_makefile(self, mode="r", buffering=None, encoding=None,
                      errors=None, newline=None):
    Backport of ``socket.makefile`` from Python 3.5.
    if not set(mode) <= {"r", "w", "b"}:
        raise ValueError(
            "invalid mode %r (only r, w, b allowed)" % (mode,)
    writing = "w" in mode
    reading = "r" in mode or not writing
    assert reading or writing
    binary = "b" in mode
    rawmode = ""
    if reading:
        rawmode += "r"
    if writing:
        rawmode += "w"
    raw = SocketIO(self, rawmode)
    self._makefile_refs += 1
    if buffering is None:
        buffering = -1
    if buffering < 0:
        buffering = io.DEFAULT_BUFFER_SIZE
    if buffering == 0:
        if not binary:
            raise ValueError("unbuffered streams must be binary")
        return raw
    if reading and writing:
        buffer = io.BufferedRWPair(raw, raw, buffering)
    elif reading:
        buffer = io.BufferedReader(raw, buffering)
    else:
        assert writing
        buffer = io.BufferedWriter(raw, buffering)
    if binary:
        return buffer
    text = io.TextIOWrapper(buffer, encoding, errors, newline)
    text.mode = mode
    return text 

def backport_makefile(
    self, mode="r", buffering=None, encoding=None, errors=None, newline=None
    Backport of ``socket.makefile`` from Python 3.5.
    if not set(mode) <= {"r", "w", "b"}:
        raise ValueError("invalid mode %r (only r, w, b allowed)" % (mode,))
    writing = "w" in mode
    reading = "r" in mode or not writing
    assert reading or writing
    binary = "b" in mode
    rawmode = ""
    if reading:
        rawmode += "r"
    if writing:
        rawmode += "w"
    raw = SocketIO(self, rawmode)
    self._makefile_refs += 1
    if buffering is None:
        buffering = -1
    if buffering < 0:
        buffering = io.DEFAULT_BUFFER_SIZE
    if buffering == 0:
        if not binary:
            raise ValueError("unbuffered streams must be binary")
        return raw
    if reading and writing:
        buffer = io.BufferedRWPair(raw, raw, buffering)
    elif reading:
        buffer = io.BufferedReader(raw, buffering)
    else:
        assert writing
        buffer = io.BufferedWriter(raw, buffering)
    if binary:
        return buffer
    text = io.TextIOWrapper(buffer, encoding, errors, newline)
    text.mode = mode
    return text 

def backport_makefile(
    self, mode="r", buffering=None, encoding=None, errors=None, newline=None
    Backport of ``socket.makefile`` from Python 3.5.
    if not set(mode) <= {"r", "w", "b"}:
        raise ValueError("invalid mode %r (only r, w, b allowed)" % (mode,))
    writing = "w" in mode
    reading = "r" in mode or not writing
    assert reading or writing
    binary = "b" in mode
    rawmode = ""
    if reading:
        rawmode += "r"
    if writing:
        rawmode += "w"
    raw = SocketIO(self, rawmode)
    self._makefile_refs += 1
    if buffering is None:
        buffering = -1
    if buffering < 0:
        buffering = io.DEFAULT_BUFFER_SIZE
    if buffering == 0:
        if not binary:
            raise ValueError("unbuffered streams must be binary")
        return raw
    if reading and writing:
        buffer = io.BufferedRWPair(raw, raw, buffering)
    elif reading:
        buffer = io.BufferedReader(raw, buffering)
    else:
        assert writing
        buffer = io.BufferedWriter(raw, buffering)
    if binary:
        return buffer
    text = io.TextIOWrapper(buffer, encoding, errors, newline)
    text.mode = mode
    return text 

def backport_makefile(self, mode="r", buffering=None, encoding=None,
                      errors=None, newline=None):
    """Backport of socket.makefile from Python 3.5."""
    if not set(mode) <= {"r", "w", "b"}:
        raise ValueError(
            "invalid mode {!r} (only r, w, b allowed)".format(mode)
    writing = "w" in mode
    reading = "r" in mode or not writing
    assert reading or writing
    binary = "b" in mode
    rawmode = ""
    if reading:
        rawmode += "r"
    if writing:
        rawmode += "w"
    raw = SocketIO(self, rawmode)
    self._makefile_refs += 1
    if buffering is None:
        buffering = -1
    if buffering < 0:
        buffering = io.DEFAULT_BUFFER_SIZE
    if buffering == 0:
        if not binary:
            raise ValueError("unbuffered streams must be binary")
        return raw
    if reading and writing:
        buffer = io.BufferedRWPair(raw, raw, buffering)
    elif reading:
        buffer = io.BufferedReader(raw, buffering)
    else:
        assert writing
        buffer = io.BufferedWriter(raw, buffering)
    if binary:
        return buffer
    text = io.TextIOWrapper(buffer, encoding, errors, newline)
    text.mode = mode
    return text 

def backport_makefile(self, mode="r", buffering=None, encoding=None,
                      errors=None, newline=None):
    Backport of ``socket.makefile`` from Python 3.5.
    if not set(mode) <= set(["r", "w", "b"]):
        raise ValueError(
            "invalid mode %r (only r, w, b allowed)" % (mode,)
    writing = "w" in mode
    reading = "r" in mode or not writing
    assert reading or writing
    binary = "b" in mode
    rawmode = ""
    if reading:
        rawmode += "r"
    if writing:
        rawmode += "w"
    raw = SocketIO(self, rawmode)
    self._makefile_refs += 1
    if buffering is None:
        buffering = -1
    if buffering < 0:
        buffering = io.DEFAULT_BUFFER_SIZE
    if buffering == 0:
        if not binary:
            raise ValueError("unbuffered streams must be binary")
        return raw
    if reading and writing:
        buffer = io.BufferedRWPair(raw, raw, buffering)
    elif reading:
        buffer = io.BufferedReader(raw, buffering)
    else:
        assert writing
        buffer = io.BufferedWriter(raw, buffering)
    if binary:
        return buffer
    text = io.TextIOWrapper(buffer, encoding, errors, newline)
    text.mode = mode
    return text 

def get_or_init_buffered_reader(self, pointer):
            if self.buffered_reader is None or self.buffered_reader.closed:
                new_file_to_read_path = self.settings.get_data_folder_path() + pointer.get_file()
                # if not exists(new_file_to_read_path):
                #     next_file = self.get_next_file(self.files, self.new_pos)
                #     if next_file is not None:
                #         new_file_to_read_path = self.settings.get_data_folder_path() + next_file
                #     else:
                #         self.buffered_reader = None
                #         return None
                self.buffered_reader = BufferedReader(FileIO(new_file_to_read_path, 'r'))
                lines_to_skip = pointer.get_line()
                if lines_to_skip > 0:
                    while self.buffered_reader.readline() is not None:
                        if lines_to_skip > 0:
                            lines_to_skip -= 1
                        else:
                            break
            return self.buffered_reader
        except IOError as e:
            log.error("Failed to initialize buffered reader! Error: %s", e)
            raise RuntimeError("Failed to initialize buffered reader!", e)
        except Exception as e:
            log.exception(e) 

def backport_makefile(self, mode="r", buffering=None, encoding=None,
                      errors=None, newline=None):
    Backport of ``socket.makefile`` from Python 3.5.
    if not set(mode) <= set(["r", "w", "b"]):
        raise ValueError(
            "invalid mode %r (only r, w, b allowed)" % (mode,)
    writing = "w" in mode
    reading = "r" in mode or not writing
    assert reading or writing
    binary = "b" in mode
    rawmode = ""
    if reading:
        rawmode += "r"
    if writing:
        rawmode += "w"
    raw = SocketIO(self, rawmode)
    self._makefile_refs += 1
    if buffering is None:
        buffering = -1
    if buffering < 0:
        buffering = io.DEFAULT_BUFFER_SIZE
    if buffering == 0:
        if not binary:
            raise ValueError("unbuffered streams must be binary")
        return raw
    if reading and writing:
        buffer = io.BufferedRWPair(raw, raw, buffering)
    elif reading:
        buffer = io.BufferedReader(raw, buffering)
    else:
        assert writing
        buffer = io.BufferedWriter(raw, buffering)
    if binary:
        return buffer
    text = io.TextIOWrapper(buffer, encoding, errors, newline)
    text.mode = mode
    return text 

def backport_makefile(self, mode="r", buffering=None, encoding=None,
                      errors=None, newline=None):
    Backport of ``socket.makefile`` from Python 3.5.
    if not set(mode) <= set(["r", "w", "b"]):
        raise ValueError(
            "invalid mode %r (only r, w, b allowed)" % (mode,)
    writing = "w" in mode
    reading = "r" in mode or not writing
    assert reading or writing
    binary = "b" in mode
    rawmode = ""
    if reading:
        rawmode += "r"
    if writing:
        rawmode += "w"
    raw = SocketIO(self, rawmode)
    self._makefile_refs += 1
    if buffering is None:
        buffering = -1
    if buffering < 0:
        buffering = io.DEFAULT_BUFFER_SIZE
    if buffering == 0:
        if not binary:
            raise ValueError("unbuffered streams must be binary")
        return raw
    if reading and writing:
        buffer = io.BufferedRWPair(raw, raw, buffering)
    elif reading:
        buffer = io.BufferedReader(raw, buffering)
    else:
        assert writing
        buffer = io.BufferedWriter(raw, buffering)
    if binary:
        return buffer
    text = io.TextIOWrapper(buffer, encoding, errors, newline)
    text.mode = mode
    return text 

def backport_makefile(self, mode="r", buffering=None, encoding=None,
                      errors=None, newline=None):
    Backport of ``socket.makefile`` from Python 3.5.
    if not set(mode) <= set(["r", "w", "b"]):
        raise ValueError(
            "invalid mode %r (only r, w, b allowed)" % (mode,)
    writing = "w" in mode
    reading = "r" in mode or not writing
    assert reading or writing
    binary = "b" in mode
    rawmode = ""
    if reading:
        rawmode += "r"
    if writing:
        rawmode += "w"
    raw = SocketIO(self, rawmode)
    self._makefile_refs += 1
    if buffering is None:
        buffering = -1
    if buffering < 0:
        buffering = io.DEFAULT_BUFFER_SIZE
    if buffering == 0:
        if not binary:
            raise ValueError("unbuffered streams must be binary")
        return raw
    if reading and writing:
        buffer = io.BufferedRWPair(raw, raw, buffering)
    elif reading:
        buffer = io.BufferedReader(raw, buffering)
    else:
        assert writing
        buffer = io.BufferedWriter(raw, buffering)
    if binary:
        return buffer
    text = io.TextIOWrapper(buffer, encoding, errors, newline)
    text.mode = mode
    return text 

def _get_text_stdin(buffer_stream):
    text_stream = _NonClosingTextIOWrapper(
        io.BufferedReader(_WindowsConsoleReader(STDIN_HANDLE)),
        'utf-16-le', 'strict', line_buffering=True)
    return ConsoleStream(text_stream, buffer_stream) 

def _get_text_stdin(buffer_stream):
    text_stream = _NonClosingTextIOWrapper(
        io.BufferedReader(_WindowsConsoleReader(STDIN_HANDLE)),
        'utf-16-le', 'strict', line_buffering=True)
    return ConsoleStream(text_stream, buffer_stream) 

def get_file_buffer(file, file_operation_mode="rb"):
    if isinstance(file, str):
        opened_file = open(file, file_operation_mode)
        buffer = opened_file.read()
        opened_file.close()
    elif isinstance(file, BufferedReader):
        buffer = file.read()
        file.close()
    elif isinstance(file, bytes):
        buffer = file
    else:
        return None
    return buffer 

def backport_makefile(self, mode="r", buffering=None, encoding=None,
                      errors=None, newline=None):
    Backport of ``socket.makefile`` from Python 3.5.
    if not set(mode) <= set(["r", "w", "b"]):
        raise ValueError(
            "invalid mode %r (only r, w, b allowed)" % (mode,)
    writing = "w" in mode
    reading = "r" in mode or not writing
    assert reading or writing
    binary = "b" in mode
    rawmode = ""
    if reading:
        rawmode += "r"
    if writing:
        rawmode += "w"
    raw = SocketIO(self, rawmode)
    self._makefile_refs += 1
    if buffering is None:
        buffering = -1
    if buffering < 0:
        buffering = io.DEFAULT_BUFFER_SIZE
    if buffering == 0:
        if not binary:
            raise ValueError("unbuffered streams must be binary")
        return raw
    if reading and writing:
        buffer = io.BufferedRWPair(raw, raw, buffering)
    elif reading:
        buffer = io.BufferedReader(raw, buffering)
    else:
        assert writing
        buffer = io.BufferedWriter(raw, buffering)
    if binary:
        return buffer
    text = io.TextIOWrapper(buffer, encoding, errors, newline)
    text.mode = mode
    return text